{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "initial_id",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:42.961852Z",
     "start_time": "2025-01-27T13:52:36.560299Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:14.138982Z",
     "iopub.status.busy": "2025-01-27T14:38:14.138815Z",
     "iopub.status.idle": "2025-01-27T14:38:16.486163Z",
     "shell.execute_reply": "2025-01-27T14:38:16.485416Z",
     "shell.execute_reply.started": "2025-01-27T14:38:14.138959Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sys.version_info(major=3, minor=10, micro=14, releaselevel='final', serial=0)\n",
      "matplotlib 3.10.0\n",
      "numpy 1.26.4\n",
      "pandas 2.2.3\n",
      "sklearn 1.6.0\n",
      "torch 2.5.1+cu124\n",
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "from tqdm.auto import tqdm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "print(sys.version_info)\n",
    "for module in mpl, np, pd, sklearn, torch:\n",
    "    print(module.__name__, module.__version__)\n",
    "\n",
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "print(device)\n",
    "\n",
    "seed = 42"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd1ebc34a8798df6",
   "metadata": {},
   "source": [
    "# 数据加载与处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "939ac15db682d46d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:43.106860Z",
     "start_time": "2025-01-27T13:52:42.962847Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:16.488089Z",
     "iopub.status.busy": "2025-01-27T14:38:16.487645Z",
     "iopub.status.idle": "2025-01-27T14:38:16.523745Z",
     "shell.execute_reply": "2025-01-27T14:38:16.523114Z",
     "shell.execute_reply.started": "2025-01-27T14:38:16.488064Z"
    }
   },
   "outputs": [],
   "source": [
    "import unicodedata\n",
    "import re\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "\n",
    "# 因为西班牙语有一些是特殊字符，所以我们需要unicode转ascii，\n",
    "# 这样值变小了，因为unicode太大\n",
    "# 将 Unicode 字符串转换为 ASCII 字符串，去除其中的重音符号\n",
    "def unicode_to_ascii(s):\n",
    "    # NFD是转换方法，把每一个字节拆开，Mn是重音，所以去除\n",
    "    return ''.join(c for c in unicodedata.normalize('NFD', s) if unicodedata.category(c) != 'Mn')\n",
    "\n",
    "# #下面我们找个样本测试一下\n",
    "# # 加u代表对字符串进行unicode编码\n",
    "# en_sentence = u\"May I borrow this book?\"\n",
    "# sp_sentence = u\"¿Puedo tomar prestado este libro?\"\n",
    "# \n",
    "# print(unicode_to_ascii(en_sentence))\n",
    "# print(unicode_to_ascii(sp_sentence))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f631de2ef6279160",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:43.114681Z",
     "start_time": "2025-01-27T13:52:43.108854Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:16.524963Z",
     "iopub.status.busy": "2025-01-27T14:38:16.524527Z",
     "iopub.status.idle": "2025-01-27T14:38:16.529286Z",
     "shell.execute_reply": "2025-01-27T14:38:16.528786Z",
     "shell.execute_reply.started": "2025-01-27T14:38:16.524937Z"
    }
   },
   "outputs": [],
   "source": [
    "def preprocess_sentence(w):\n",
    "    # 变为小写，去掉多余的空格，变成小写，id少一些\n",
    "    w = unicode_to_ascii(w.lower().strip())\n",
    "\n",
    "    # 特定标点符号（?.!,¿）前后添加空格\n",
    "    # eg: \"he is a boy.\" => \"he is a boy . \"\n",
    "    # 这样可以使得模型更容易分辨单词和标点符号\n",
    "    w = re.sub(r\"([?.!,¿])\", r\" \\1 \", w)\n",
    "    # 因为可能有多余空格，替换为一个空格，所以处理一下\n",
    "    w = re.sub(r'[\" \"]+', \" \", w)\n",
    "    # 将字符串 w 中所有非字母字符（包括标点符号 ?.!,¿）替换为空格\n",
    "    # [^...] 表示匹配不在括号内的字符。\n",
    "    # a-zA-Z 匹配所有大小写字母。\n",
    "    # ?.!,¿ 匹配特定的标点符号。\n",
    "    #+ 表示匹配一个或多个连续字符。\n",
    "    w = re.sub(r\"[^a-zA-Z?.!,¿]+\", \" \", w)\n",
    "\n",
    "    #  先去除末尾空白字符，再去除首尾空白字符。\n",
    "    w = w.rstrip().strip()\n",
    "    return w\n",
    "\n",
    "# print(preprocess_sentence(en_sentence))\n",
    "# print(preprocess_sentence(sp_sentence))\n",
    "# print(preprocess_sentence(sp_sentence).encode('utf-8'))  #¿是占用两个字节的"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c36ecfc4cba8609b",
   "metadata": {},
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c78b45975b61bd8f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:43.119627Z",
     "start_time": "2025-01-27T13:52:43.116677Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:16.530451Z",
     "iopub.status.busy": "2025-01-27T14:38:16.529954Z",
     "iopub.status.idle": "2025-01-27T14:38:16.532872Z",
     "shell.execute_reply": "2025-01-27T14:38:16.532303Z",
     "shell.execute_reply.started": "2025-01-27T14:38:16.530427Z"
    }
   },
   "outputs": [],
   "source": [
    "# #zip例子\n",
    "# a = [[1, 2], [4, 5], [7, 8]]\n",
    "# # *a 是 Python 的解包操作符，将 a 解包为 3 个子列表：\n",
    "# # zip 函数将解包后的子列表按位置组合\n",
    "# zipped = list(zip(*a))\n",
    "# print(zipped)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4f0e1f6b9133f7cc",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:43.126777Z",
     "start_time": "2025-01-27T13:52:43.122623Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:16.533996Z",
     "iopub.status.busy": "2025-01-27T14:38:16.533503Z",
     "iopub.status.idle": "2025-01-27T14:38:16.536328Z",
     "shell.execute_reply": "2025-01-27T14:38:16.535782Z",
     "shell.execute_reply.started": "2025-01-27T14:38:16.533972Z"
    }
   },
   "outputs": [],
   "source": [
    "# split_index1 = np.random.choice(a=[\"train\", \"test\"],\n",
    "#                                 replace=True, p=[0.9, 0.1], size=100)\n",
    "# print(len(split_index1))\n",
    "# print(\"-\" * 50)\n",
    "# split_index1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4bc7f9f4cb98ac0f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:43.582409Z",
     "start_time": "2025-01-27T13:52:43.128774Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:16.537354Z",
     "iopub.status.busy": "2025-01-27T14:38:16.536958Z",
     "iopub.status.idle": "2025-01-27T14:38:16.905319Z",
     "shell.execute_reply": "2025-01-27T14:38:16.904743Z",
     "shell.execute_reply.started": "2025-01-27T14:38:16.537333Z"
    }
   },
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "\n",
    "class LangPairDataset(Dataset):\n",
    "    fpath = Path(r\"./data_spa_en/spa.txt\")  #数据文件路径\n",
    "    cache_path = Path(r\"./.cache/lang_pair.npy\")  #缓存文件路径 \n",
    "    # 按照9:1划分训练集和测试集\n",
    "    split_index = np.random.choice(a=[\"train\", \"test\"], replace=True, p=[0.9, 0.1], size=118964)\n",
    "\n",
    "    def __init__(self, mode=\"train\", cache=False):\n",
    "        # if cache or not self.cache_path.exists() 表示：\n",
    "        # 如果启用缓存（cache 为 True），或者缓存文件不存在，则条件成立。\n",
    "        if cache or not self.cache_path.exists():\n",
    "            # parent 属性返回 self.cache_path 的父目录。\n",
    "            # mkdir 是 Path 对象的方法，用于创建目录。\n",
    "            # parents=True 表示递归创建父目录（如果父目录不存在）。\n",
    "            # exist_ok=True 表示如果目录已存在，不会抛出错误。\n",
    "            self.cache_path.parent.mkdir(exist_ok=True, parents=True)\n",
    "            with open(self.fpath, \"r\", encoding=\"utf-8\") as fd:\n",
    "                # 从文件中读取所有行，并将其存储在一个列表中\n",
    "                lines = fd.readlines()\n",
    "                # 将 lines 中的每一行按制表符 \\t 分割，并对每个分割后的部分调用 preprocess_sentence 函数进行预处理\n",
    "                lang_pair = [[preprocess_sentence(w) for w in l.split(\"\\t\")] for l in lines]\n",
    "                # 分离出目标语言和源语言\n",
    "                trg, src = zip(*lang_pair)\n",
    "                trg = np.array(trg)\n",
    "                src = np.array(src)\n",
    "                # 保存为npy文件,方便下次直接读取,不用再处理\n",
    "                np.save(self.cache_path, {\"trg\": trg, \"src\": src})\n",
    "        else:\n",
    "            # 从 .npy 文件中加载数据，并将其转换为 Python 字典（或其他对象）\n",
    "            # np.load 是 NumPy 提供的函数，用于从 .npy 或 .npz 文件中加载数据。\n",
    "            # allow_pickle=True 表示允许从 .npy 文件中加载 Python 对象。\n",
    "            # .item() 将其转换为 Python 对象（如字典、列表、字典等）\n",
    "            lang_pair = np.load(self.cache_path, allow_pickle=True).item()\n",
    "            trg = lang_pair[\"trg\"]\n",
    "            src = lang_pair[\"src\"]\n",
    "\n",
    "        # 按照index拿到训练集的 标签语言 --英语\n",
    "        self.trg = trg[self.split_index == mode]\n",
    "        # 按照index拿到训练集的源语言 --西班牙\n",
    "        self.src = src[self.split_index == mode]\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        return self.src[index], self.trg[index]\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.src)\n",
    "\n",
    "\n",
    "train_ds = LangPairDataset(mode=\"train\")\n",
    "test_ds = LangPairDataset(mode=\"test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "83a0eec9d0d9e91d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:43.587926Z",
     "start_time": "2025-01-27T13:52:43.583404Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:16.907716Z",
     "iopub.status.busy": "2025-01-27T14:38:16.907374Z",
     "iopub.status.idle": "2025-01-27T14:38:16.911064Z",
     "shell.execute_reply": "2025-01-27T14:38:16.910458Z",
     "shell.execute_reply.started": "2025-01-27T14:38:16.907693Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "source:si quieres sonar como un hablante nativo , debes estar dispuesto a practicar diciendo la misma frase una y otra vez de la misma manera en que un musico de banjo practica el mismo fraseo una y otra vez hasta que lo puedan tocar correctamente y en el tiempo esperado .\n",
      "target:if you want to sound like a native speaker , you must be willing to practice saying the same sentence over and over in the same way that banjo players practice the same phrase over and over until they can play it correctly and at the desired tempo .\n"
     ]
    }
   ],
   "source": [
    "print(\"source:{}\\ntarget:{}\".format(*train_ds[-1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "906e6d420362b515",
   "metadata": {},
   "source": [
    "## Tokenizer\n",
    "\n",
    "这里有两种处理方式，分别对应着 encoder 和 decoder 的 word embedding 是否共享，这里实现不共享的方案。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ae72149b871c746d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:44.281040Z",
     "start_time": "2025-01-27T13:52:43.589438Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:16.911924Z",
     "iopub.status.busy": "2025-01-27T14:38:16.911662Z",
     "iopub.status.idle": "2025-01-27T14:38:17.528256Z",
     "shell.execute_reply": "2025-01-27T14:38:17.527638Z",
     "shell.execute_reply.started": "2025-01-27T14:38:16.911903Z"
    }
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "\n",
    "def get_word_idx(ds, mode=\"src\", threshold=2):\n",
    "    #载入词表，看下词表长度，词表就像英语字典\n",
    "    word2idx = {\n",
    "        \"[PAD]\": 0,  # 填充 token\n",
    "        \"[BOS]\": 1,  # begin of sentence\n",
    "        \"[UNK]\": 2,  # 未知 token\n",
    "        \"[EOS]\": 3,  # end of sentence\n",
    "    }\n",
    "    idx2word = {value: key for key, value in word2idx.items()}\n",
    "    index = len(idx2word)\n",
    "    # 出现次数低于此的token舍弃\n",
    "    threshold = 1\n",
    "    # 如果数据集有很多个G，那是用for循环的，不能' '.join\n",
    "    word_list = \" \".join([pair[0 if mode == \"src\" else 1] for pair in ds]).split()\n",
    "    # print(type(word_list)) # <class 'list'>\n",
    "    # 统计词频,counter类似字典，key是单词，value是出现次数\n",
    "    counter = Counter(word_list)\n",
    "    # print(\"word count:\", len(counter))\n",
    "\n",
    "    for token, count in counter.items():\n",
    "        # 出现次数大于阈值的token加入词表\n",
    "        if count >= threshold:\n",
    "            word2idx[token] = index  # 加入词表\n",
    "            idx2word[index] = token  # 加入反向词典\n",
    "            index += 1\n",
    "\n",
    "    return word2idx, idx2word\n",
    "\n",
    "\n",
    "# 源语言词表  西班牙语\n",
    "src_word2idx, src_idx2word = get_word_idx(train_ds, mode=\"src\")\n",
    "# 目标语言词表 英语\n",
    "trg_word2idx, trg_idx2word = get_word_idx(train_ds, mode=\"trg\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1585f8ee2b3174b7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:44.290344Z",
     "start_time": "2025-01-27T13:52:44.282033Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:17.529253Z",
     "iopub.status.busy": "2025-01-27T14:38:17.528960Z",
     "iopub.status.idle": "2025-01-27T14:38:17.538727Z",
     "shell.execute_reply": "2025-01-27T14:38:17.538176Z",
     "shell.execute_reply.started": "2025-01-27T14:38:17.529230Z"
    }
   },
   "outputs": [],
   "source": [
    "class Tokenizer:\n",
    "    def __init__(self, word2idx, idx2word, max_length=500,\n",
    "                 pad_idx=0, bos_idx=1, eos_idx=3, unk_idx=2):\n",
    "        self.word2idx = word2idx\n",
    "        self.idx2word = idx2word\n",
    "        self.max_length = max_length\n",
    "        self.pad_idx = pad_idx\n",
    "        self.bos_idx = bos_idx\n",
    "        self.eos_idx = eos_idx\n",
    "        self.unk_idx = unk_idx\n",
    "\n",
    "    def encode(self, text_list, padding_first=False, add_bos=True, add_eos=True,\n",
    "               return_mask=False):\n",
    "        # 如果padding_first == True，则padding加载前面，否则加载后面\n",
    "        # return_mask: 是否返回mask(掩码），mask用于指示哪些是padding的，哪些是真实的token \n",
    "        # 若启动bos_eos，则在句首和句尾分别添加bos和eos，则 add_bos=True, add_eos=True即都为1\n",
    "        max_length = min(self.max_length, add_bos + add_eos + max([len(text) for text in text_list]))\n",
    "        indices_list = []\n",
    "        for text in text_list:\n",
    "            # 如果词表中没有这个词，就用unk_idx代替，indices是一个list,里面是每个词的index,也就是一个样本的index\n",
    "            indices = [self.word2idx.get(word, self.unk_idx) for word in text[:max_length - add_eos - add_bos]]\n",
    "            if add_bos:\n",
    "                indices = [self.bos_idx] + indices\n",
    "            if add_eos:\n",
    "                indices = indices + [self.eos_idx]\n",
    "            if padding_first:  # padding加载前面，超参可以调\n",
    "                indices = [self.pad_idx] * (max_length - len(indices)) + indices\n",
    "            else:  # padding加载后面\n",
    "                indices = indices + [self.pad_idx] * (max_length - len(indices))\n",
    "            indices_list.append(indices)\n",
    "        input_ids = torch.tensor(indices_list)  # 转换为tensor\n",
    "        # mask是一个和input_ids一样大小的tensor，0代表token，1代表padding，mask用于去除padding的影响\n",
    "        masks = (input_ids == self.pad_idx).to(dtype=torch.float64)\n",
    "        return input_ids if not return_mask else (input_ids, masks)\n",
    "\n",
    "    def decode(self, indices_list, remove_bos=True, remove_eos=True, remove_pad=True, split=False):\n",
    "        text_list = []\n",
    "        for indices in indices_list:\n",
    "            text = []\n",
    "            for index in indices:\n",
    "                # 如果词表中没有这个词，就用unk_idx代替\n",
    "                word = self.idx2word.get(index, \"[UNK]\")\n",
    "                if remove_bos and word == \"[BOS]\":\n",
    "                    continue\n",
    "                if remove_eos and word == \"[EOS]\":  # 如果到达eos，就结束\n",
    "                    break\n",
    "                if remove_pad and word == \"[PAD]\":  # 如果到达pad，就结束\n",
    "                    break\n",
    "                text.append(word)  # 单词添加到列表中\n",
    "            # 把列表中的单词拼接，变为一个句子\n",
    "            text_list.append(\" \".join(text) if not split else text)\n",
    "        return text_list\n",
    "\n",
    "\n",
    "#两个相对于1个toknizer的好处是embedding的参数量减少\n",
    "# 源语言tokenizer\n",
    "src_tokenizer = Tokenizer(word2idx=src_word2idx, idx2word=src_idx2word)\n",
    "# 目标语言tokenizer\n",
    "trg_tokenizer = Tokenizer(word2idx=trg_word2idx, idx2word=trg_idx2word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "602e9cd5a872b5a5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:44.296339Z",
     "start_time": "2025-01-27T13:52:44.292339Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:17.539892Z",
     "iopub.status.busy": "2025-01-27T14:38:17.539419Z",
     "iopub.status.idle": "2025-01-27T14:38:17.542313Z",
     "shell.execute_reply": "2025-01-27T14:38:17.541862Z",
     "shell.execute_reply.started": "2025-01-27T14:38:17.539870Z"
    }
   },
   "outputs": [],
   "source": [
    "# # trg_tokenizer.encode([[\"hello\"], [\"hello\", \"world\"]], add_bos=True, add_eos=False,return_mask=True)\n",
    "# raw_text = [\"hello world\".split(), \"tokenize text datas with batch\".split(), \"this is a test\".split()]\n",
    "# indices,mask = trg_tokenizer.encode(raw_text, padding_first=False, add_bos=True, add_eos=True,return_mask=True)\n",
    "# \n",
    "# print(\"raw text\"+'-'*10)\n",
    "# for raw in raw_text:\n",
    "#     print(raw)\n",
    "# print(\"mask\"+'-'*10)\n",
    "# for m in mask:\n",
    "#     print(m)\n",
    "# print(\"indices\"+'-'*10)\n",
    "# for index in indices:\n",
    "#     print(index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "1a5102956b2ec3f1",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:44.302231Z",
     "start_time": "2025-01-27T13:52:44.298340Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:17.543317Z",
     "iopub.status.busy": "2025-01-27T14:38:17.542911Z",
     "iopub.status.idle": "2025-01-27T14:38:17.545519Z",
     "shell.execute_reply": "2025-01-27T14:38:17.545061Z",
     "shell.execute_reply.started": "2025-01-27T14:38:17.543296Z"
    }
   },
   "outputs": [],
   "source": [
    "# decode_text = trg_tokenizer.decode(indices.tolist(), remove_bos=False, remove_eos=False, remove_pad=False)\n",
    "# print(\"decode text\"+'-'*10)\n",
    "# for decode in decode_text:\n",
    "#     print(decode)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "df29f25d3c740e46",
   "metadata": {},
   "source": [
    "## DataLoader\n",
    "\n",
    "encoder设置mask的原因：在softmax计算时，padding的位置计算结果趋近于0\n",
    "\n",
    "decoder设置mask的原因：在计算loss时，padding的位置的loss不参与计算"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "caca3658e2a7a115",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:44.311318Z",
     "start_time": "2025-01-27T13:52:44.304224Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:17.546378Z",
     "iopub.status.busy": "2025-01-27T14:38:17.546151Z",
     "iopub.status.idle": "2025-01-27T14:38:17.551845Z",
     "shell.execute_reply": "2025-01-27T14:38:17.550914Z",
     "shell.execute_reply.started": "2025-01-27T14:38:17.546359Z"
    }
   },
   "outputs": [],
   "source": [
    "def collate_fn(batch):\n",
    "    # 取batch内第0列进行分词，赋给src_words\n",
    "    src_words = [pair[0].split() for pair in batch]\n",
    "    # 取batch内第1列进行分词，赋给trg_words\n",
    "    trg_words = [pair[1].split() for pair in batch]\n",
    "\n",
    "    # [PAD] [BOS] src [EOS]\n",
    "    encoder_inputs, encoder_inputs_mask = src_tokenizer.encode(src_words, padding_first=True, add_bos=True,\n",
    "                                                               add_eos=True, return_mask=True)\n",
    "\n",
    "    # 目标语言 [BOS] trg [PAD]\n",
    "    decoder_inputs = trg_tokenizer.encode(trg_words, padding_first=False, add_bos=True, add_eos=False,\n",
    "                                          return_mask=False)\n",
    "\n",
    "    # 用于后续计算loss\n",
    "    # trg [EOS] [PAD]\n",
    "    decoder_labels, decoder_labels_mask = trg_tokenizer.encode(trg_words, padding_first=False, add_bos=False,\n",
    "                                                               add_eos=True, return_mask=True)\n",
    "\n",
    "    return {\n",
    "        \"encoder_inputs\": encoder_inputs.to(device),\n",
    "        \"encoder_inputs_mask\": encoder_inputs_mask.to(device),\n",
    "        \"decoder_inputs\": decoder_inputs.to(device),\n",
    "        \"decoder_labels\": decoder_labels.to(device),\n",
    "        # mask用于去除padding的影响，计算loss时用\n",
    "        \"decoder_labels_mask\": decoder_labels_mask.to(device),\n",
    "    }  # 当返回的数据较多时，用dict返回比较合理\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "252c37d675f553a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:44.317518Z",
     "start_time": "2025-01-27T13:52:44.313306Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:17.552756Z",
     "iopub.status.busy": "2025-01-27T14:38:17.552513Z",
     "iopub.status.idle": "2025-01-27T14:38:17.555084Z",
     "shell.execute_reply": "2025-01-27T14:38:17.554624Z",
     "shell.execute_reply.started": "2025-01-27T14:38:17.552734Z"
    }
   },
   "outputs": [],
   "source": [
    "# for batch in train_loader:\n",
    "#     for key, value in batch.items():\n",
    "#         print(key)\n",
    "#         print(value.shape)\n",
    "#         print(\"-\"*50)\n",
    "#     break    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "929678e26f67eccc",
   "metadata": {},
   "source": [
    "# 定义模型"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19fb8fb3385416ab",
   "metadata": {},
   "source": [
    "## Encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "998ca06a810799f6",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:44.327992Z",
     "start_time": "2025-01-27T13:52:44.321515Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:17.555811Z",
     "iopub.status.busy": "2025-01-27T14:38:17.555617Z",
     "iopub.status.idle": "2025-01-27T14:38:17.560264Z",
     "shell.execute_reply": "2025-01-27T14:38:17.559729Z",
     "shell.execute_reply.started": "2025-01-27T14:38:17.555790Z"
    }
   },
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_dim=256, hidden_dim=1024, num_layers=1):\n",
    "        super().__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
    "        self.gru = nn.GRU(embedding_dim, hidden_dim, num_layers=num_layers, batch_first=True)\n",
    "\n",
    "    def forward(self, encoder_inputs):\n",
    "        # encoder_inputs.shape = [batch size, sequence_length]\n",
    "        embeds = self.embedding(encoder_inputs)\n",
    "        # embeds.shape = [batch size, sequence_length, embedding_dim]\n",
    "        seq_output, hidden = self.gru(embeds)\n",
    "        # seq_output.shape = [batch size, sequence_length, hidden_dim]\n",
    "        # hidden.shape = [num_layers, batch size, hidden_dim]\n",
    "        return seq_output, hidden\n",
    "        # encoder_outputs[:,-1,:]==hidden[-1,:,:]  全是True 说明输出和hidden是相同的"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4862d8fe85734897",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:44.334614Z",
     "start_time": "2025-01-27T13:52:44.329989Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:17.561184Z",
     "iopub.status.busy": "2025-01-27T14:38:17.560951Z",
     "iopub.status.idle": "2025-01-27T14:38:17.563832Z",
     "shell.execute_reply": "2025-01-27T14:38:17.563321Z",
     "shell.execute_reply.started": "2025-01-27T14:38:17.561164Z"
    }
   },
   "outputs": [],
   "source": [
    "# #把上面的Encoder写一个例子，看看输出的shape\n",
    "# encoder = Encoder(vocab_size=100, embedding_dim=256, hidden_dim=1024, num_layers=4)\n",
    "# # 生成一个形状为 (2, 50) 的随机整数张量，其中每个元素的取值范围是 [0, 100)\n",
    "# encoder_inputs = torch.randint(0, 100, (2, 50)) # [2,50]\n",
    "# encoder_outputs, hidden = encoder(encoder_inputs)\n",
    "# print(encoder_outputs.shape) # [2,50,1024]\n",
    "# print(hidden.shape) # [4,2,1024]\n",
    "# print(encoder_outputs[:,-1,:]) # 取最后一个时间步的输出\n",
    "# print(hidden[-1,:,:]) #取最后一层的hidden\n",
    "# print(encoder_outputs[:,-1,:]==hidden[-1,:,:]) # 全是True 说明输出和hidden是相同的"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "96f2a739975ee635",
   "metadata": {},
   "source": [
    "## BahdanauAttention公式\n",
    "score = FC(tanh(FC(EO) + FC(H))) 其中FC(EO)的FC是Wk,FC(H)的FC是Wq,最外面的FC是V \n",
    "\n",
    "attention_weights = softmax(score, axis = 1)  \n",
    "\n",
    "context = sum(attention_weights * EO, axis = 1) #对EO做加权求和，得到上下文向量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "df0075e30e05dd4a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:44.344148Z",
     "start_time": "2025-01-27T13:52:44.336609Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:17.564612Z",
     "iopub.status.busy": "2025-01-27T14:38:17.564387Z",
     "iopub.status.idle": "2025-01-27T14:38:17.570321Z",
     "shell.execute_reply": "2025-01-27T14:38:17.569805Z",
     "shell.execute_reply.started": "2025-01-27T14:38:17.564592Z"
    }
   },
   "outputs": [],
   "source": [
    "class BahdanauAttention(nn.Module):\n",
    "    def __init__(self, hidden_dim=1024):\n",
    "        super().__init__()\n",
    "        # 对keys做运算，encoder的输出EO\n",
    "        self.Wk = nn.Linear(hidden_dim, hidden_dim)\n",
    "        # 对query做运算，decoder的隐藏状态\n",
    "        self.Wq = nn.Linear(hidden_dim, hidden_dim)\n",
    "        self.V = nn.Linear(hidden_dim, 1)\n",
    "\n",
    "    def forward(self, query, keys, values, attn_mask=None):\n",
    "        # query: hidden state，是decoder的隐藏状态，shape = [batch size,hidden_dim]\n",
    "        # keys: EO [batch size, sequence length, hidden_dim]\n",
    "        # values: EO  [batch size, sequence length, hidden_dim]\n",
    "        # attn_mask:[batch size, sequence length],这里是encoder_inputs_mask\n",
    "\n",
    "        # query.shape = [batch size, hidden_dim] -->通过unsqueeze(-2)增加维度 [batch size, 1, hidden_dim]\n",
    "        # keys.shape = [batch size, sequence length, hidden_dim]\n",
    "        # values.shape = [batch size, sequence length, hidden_dim]\n",
    "        score = self.V(F.tanh(self.Wk(keys) + self.Wq(query.unsqueeze(-2))))\n",
    "        # score.shape = [batch size, sequence length, 1]\n",
    "\n",
    "        #这个mask是encoder_inputs_mask，用来mask掉padding的部分,让padding部分socres为0\n",
    "        if attn_mask is not None:\n",
    "            # 在最后增加一个维度，\n",
    "            # [batch size, sequence length] --> [batch size, sequence length, 1]\n",
    "            attn_mask = (attn_mask.unsqueeze(-1)) * -1e16  # 0还是0，1变为-1e16\n",
    "            # 即填充Pad的区域为-1e16，softmax后，这些位置的权重为0，不会影响输出\n",
    "            score += attn_mask\n",
    "        score = F.softmax(score, dim=-2)  # 对每一个词的score做softmax\n",
    "        # score.shape = [batch size, sequence length, 1]\n",
    "        #对每一个词的score和对应的value做乘法，然后在seq_len维度上求和，得到context_vector\n",
    "        context_vector = torch.mul(score, values).sum(dim=-2)\n",
    "        # context_vector.shape = [batch size, hidden_dim]\n",
    "        return context_vector, score\n",
    "        # score: 注意力权重，用于画图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "6f6fc98dab908ecf",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:44.350311Z",
     "start_time": "2025-01-27T13:52:44.346143Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:17.571243Z",
     "iopub.status.busy": "2025-01-27T14:38:17.570949Z",
     "iopub.status.idle": "2025-01-27T14:38:17.573568Z",
     "shell.execute_reply": "2025-01-27T14:38:17.573073Z",
     "shell.execute_reply.started": "2025-01-27T14:38:17.571222Z"
    }
   },
   "outputs": [],
   "source": [
    "# #tensor矩阵相乘\n",
    "# a = torch.randn(2, 3)\n",
    "# b = torch.randn(2, 3)\n",
    "# c = torch.mul(a, b) #增加维度\n",
    "# print(c.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "bc236d1b2f0f089a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:44.355408Z",
     "start_time": "2025-01-27T13:52:44.351307Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:17.574400Z",
     "iopub.status.busy": "2025-01-27T14:38:17.574228Z",
     "iopub.status.idle": "2025-01-27T14:38:17.576879Z",
     "shell.execute_reply": "2025-01-27T14:38:17.576365Z",
     "shell.execute_reply.started": "2025-01-27T14:38:17.574381Z"
    }
   },
   "outputs": [],
   "source": [
    "# #把上面的BahdanauAttention写一个例子，看看输出的shape\n",
    "# attention = BahdanauAttention(hidden_dim=1024)\n",
    "# query = torch.randn(2, 1024) #Decoder的隐藏状态\n",
    "# keys = torch.randn(2, 50, 1024) #EO\n",
    "# values = torch.randn(2, 50, 1024) #EO\n",
    "# attn_mask = torch.randint(0, 2, (2, 50))\n",
    "# context_vector, scores = attention(query, keys, values, attn_mask)\n",
    "# print(context_vector.shape)\n",
    "# print(scores.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1517bf2e93e55643",
   "metadata": {},
   "source": [
    "## Decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "731b9a63e710aaad",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:44.363969Z",
     "start_time": "2025-01-27T13:52:44.356399Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:17.577737Z",
     "iopub.status.busy": "2025-01-27T14:38:17.577421Z",
     "iopub.status.idle": "2025-01-27T14:38:17.584459Z",
     "shell.execute_reply": "2025-01-27T14:38:17.583873Z",
     "shell.execute_reply.started": "2025-01-27T14:38:17.577717Z"
    }
   },
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_dim=256, hidden_dim=1024, num_layers=1):\n",
    "        super().__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
    "        self.gru = nn.GRU(embedding_dim + hidden_dim, hidden_dim, num_layers=num_layers, batch_first=True)\n",
    "        # 最后分类,词典大小是多少，就输出多少个分类\n",
    "        self.fc = nn.Linear(hidden_dim, vocab_size)\n",
    "        # 定义一个 Dropout 层，随机丢弃 60% 的神经元输出\n",
    "        self.dropout = nn.Dropout(0.6)\n",
    "        # 定义一个 BahdanauAttention 层,得到的context_vector\n",
    "        self.attention = BahdanauAttention(hidden_dim=hidden_dim)\n",
    "\n",
    "    def forward(self, decoder_inputs, hidden, encoder_outputs, attn_mask=None):\n",
    "        # attn_mask是encoder_inputs_mask,用来mask掉padding的部分,让padding部分socres为0\n",
    "\n",
    "        # decoder_inputs.shape = [batch size, 1]\n",
    "        assert len(decoder_inputs.shape) == 2 and decoder_inputs.shape[\n",
    "            -1] == 1, f\"decoder_input.shape = {decoder_inputs.shape} is not valid\"\n",
    "\n",
    "        # hidden.shape = [batch size, hidden_dim]，decoder_hidden,\n",
    "        # 而第一次使用的是encoder的hidden\n",
    "        assert len(hidden.shape) == 2, f\"hidden.shape = {hidden.shape} is not valid\"\n",
    "\n",
    "        # encoder_outputs.shape = [batch size, sequence length, hidden_dim]\n",
    "        assert len(encoder_outputs.shape) == 3, f\"encoder_outputs.shape = {encoder_outputs.shape} is not valid\"\n",
    "\n",
    "        # context_vector.shape = [batch_size, hidden_dim]\n",
    "        context_vector, attention_score = self.attention(query=hidden, keys=encoder_outputs, values=encoder_outputs,\n",
    "                                                         attn_mask=attn_mask)\n",
    "\n",
    "        # decoder_input.shape = [batch size, 1]\n",
    "        embeds = self.embedding(decoder_inputs)\n",
    "        # embeds.shape = [batch size, 1, embedding_dim]\n",
    "        # context_vector.shape = [batch size, hidden_dim] -->unsqueeze(-2)增加维度 [batch size, 1, hidden_dim]\n",
    "        embeds = torch.cat([context_vector.unsqueeze(-2), embeds], dim=-1)\n",
    "        # embeds.shape = [batch size, 1, embedding_dim+hidden_dim]\n",
    "        seq_output, hidden = self.gru(embeds)\n",
    "        # seq_output.shape = [batch size, 1, hidden_dim]\n",
    "        logits = self.fc(seq_output)\n",
    "        # logits.shape = [batch size, 1, vocab_size]\n",
    "        # attention_score.shape = [batch size, sequence length, 1]\n",
    "        return logits, hidden, attention_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "70d7459e2ecbd5de",
   "metadata": {},
   "source": [
    "## Seq2Seq模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "514f7b41823ae373",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:44.375072Z",
     "start_time": "2025-01-27T13:52:44.364965Z"
    },
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:17.585559Z",
     "iopub.status.busy": "2025-01-27T14:38:17.585258Z",
     "iopub.status.idle": "2025-01-27T14:38:17.595983Z",
     "shell.execute_reply": "2025-01-27T14:38:17.595338Z",
     "shell.execute_reply.started": "2025-01-27T14:38:17.585539Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class Sequence2Sequence(nn.Module):\n",
    "    def __init__(\n",
    "            self,\n",
    "            src_vocab_size,  # 输入词典大小\n",
    "            trg_vocab_size,  # 输出词典大小\n",
    "            encoder_embedding_dim=256,\n",
    "            #encoder_hidden_dim和decoder_hidden_dim必须相同，是因为BahdanauAttention设计的\n",
    "            encoder_hidden_dim=1024,\n",
    "            encoder_num_layers=1,\n",
    "            decoder_embedding_dim=256,\n",
    "            decoder_hidden_dim=1024,\n",
    "            decoder_num_layers=1,\n",
    "            bos_idx=1,\n",
    "            eos_idx=3,\n",
    "            max_length=512,\n",
    "    ):\n",
    "        super().__init__()\n",
    "        self.bos_idx = bos_idx\n",
    "        self.eos_idx = eos_idx\n",
    "        self.max_length = max_length\n",
    "        self.encoder = Encoder(src_vocab_size, embedding_dim=encoder_embedding_dim, hidden_dim=encoder_hidden_dim,\n",
    "                               num_layers=encoder_num_layers)\n",
    "        self.decoder = Decoder(trg_vocab_size, embedding_dim=decoder_embedding_dim, hidden_dim=decoder_hidden_dim,\n",
    "                               num_layers=decoder_num_layers)\n",
    "\n",
    "    def forward(self, encoder_inputs, decoder_inputs, attn_mask=None):\n",
    "        \"\"\"\n",
    "        用于训练\n",
    "        (1)根据encoder_inputs计算出eneoder_ouyputs和hidden，用与计算上下文context_vector和scores\n",
    "        (2)然后根据decoder_inputs,context_vector和hidden计算出decoder_outputs和新的hidden，decoder_outputs能算出预测的logits\n",
    "        (3)最后将logits拼接、score拼接，作为模型的输出\n",
    "        注意：训练是一个元素一个元素计算，decoder_inputs其实就是训练样本的正确结果，即logit是由样本提问和回答结果计算得到的，所以训练时，decoder_inputs是正确答案，而非模型预测的结果。\n",
    "        \"\"\"\n",
    "        # encoding\n",
    "        encoder_outputs, hidden = self.encoder(encoder_inputs)\n",
    "        # decoding\n",
    "        batch_size, seq_len = decoder_inputs.shape\n",
    "        logits_list = []\n",
    "        scores_list = []\n",
    "        for i in range(seq_len):  # 串行训练\n",
    "            # 每次迭代生成一个时间步的预测，存储在 logits_list 中，并且记录注意力分数（如果有的话）在 scores_list 中，最后将预测的logits和注意力分数拼接并返回。\n",
    "            logits, hidden, scores = self.decoder(\n",
    "                decoder_inputs[:, i:i + 1],  # 取第i个时间步的输入作为decoder的输入\n",
    "                # 取最后一层的hidden，第一个时间步时，hidden是encoder的hidden，第二个时间步时，hidden是decoder的hidden\n",
    "                hidden[-1],\n",
    "                encoder_outputs,\n",
    "                attn_mask=attn_mask,\n",
    "            )\n",
    "            # logits.shape = [batch size, 1,vocab_size]\n",
    "            # scores.shape = [batch size, sequence length, 1]\n",
    "            logits_list.append(logits)  # 记录预测的logits，用于计算损失\n",
    "            scores_list.append(scores)  # 记录注意力分数，用于画图\n",
    "        # [batch size, seq_len, vocab_size], [batch size, seq_len, seq_length]\n",
    "        return torch.cat(logits_list, dim=-2), torch.cat(scores_list, dim=-1)\n",
    "\n",
    "    @torch.no_grad()  # 装饰器，禁止梯度计算\n",
    "    def infer(self, encoder_inputs, attn_mask=None):\n",
    "        \"\"\"\n",
    "        用于预测，\n",
    "        此时，decoder_inputs是开始标记，\n",
    "        模型根据encoder_inputs(即提问)计算出eceoder_ouyputs和hidden，\n",
    "        用与计算上下文context_vector和scores，\n",
    "        然后根据context_vector和hidden生成第一个词，\n",
    "        作为下一个decoder_inputs，用于生成第二个词，作为新的decoder_inputs，以此类推，直到生成结束标记 eos_idx 或达到最大长度 max_length。\n",
    "        \"\"\"\n",
    "        # infer用于预测\n",
    "\n",
    "        # encoder_input.shape = [1, sequence length],这只支持batch_size=1\n",
    "        # encoding\n",
    "        encoder_outputs, hidden = self.encoder(encoder_inputs)\n",
    "\n",
    "        # decoding\n",
    "        # 创建一个形状为 (1, 1) 的张量，并将其数据类型转换为 torch.int64\n",
    "        # shape为[1,1]，内容为开始标记\n",
    "        decoder_inputs = torch.Tensor([self.bos_idx]).reshape(1, 1).to(dtype=torch.int64)\n",
    "        decoder_pred = None\n",
    "        pred_list = []  # 预测序列\n",
    "        scores_list = []\n",
    "        # 从开始标记 bos_idx 开始，迭代地生成序列，直到生成结束标记 eos_idx 或达到最大长度 max_length。\n",
    "        for _ in range(self.max_length):\n",
    "            logits, hidden, scores = self.decoder(\n",
    "                decoder_inputs,\n",
    "                hidden[-1],\n",
    "                encoder_outputs,\n",
    "                attn_mask=attn_mask,\n",
    "            )\n",
    "            # logits.shape = [1, 1,vocab_size]\n",
    "            decoder_pred = logits.argmax(dim=-1)\n",
    "            # decoder_pred.shape = [1, 1]\n",
    "            decoder_inputs = decoder_pred\n",
    "            # reshape(-1):从(1,1)变为（1）标量\n",
    "            pred_list.append(decoder_pred.reshape(-1).item())  # 记录预测的词\n",
    "            # scores.shape = [1, sequence length, 1]\n",
    "            scores_list.append(scores)  # 记录注意力分数，用于画图\n",
    "\n",
    "            # 如果生成结束标记，则停止迭代\n",
    "            if decoder_pred == self.eos_idx:\n",
    "                break\n",
    "\n",
    "        # cat(scores_list,dim=-1):[1, sequence length, sequence length]\n",
    "        return pred_list, torch.cat(scores_list, dim=-1)  # 返回预测序列和注意力分数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "d119467be4a8eb18",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:45.185268Z",
     "start_time": "2025-01-27T13:52:44.376068Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:17.596931Z",
     "iopub.status.busy": "2025-01-27T14:38:17.596670Z",
     "iopub.status.idle": "2025-01-27T14:38:18.046402Z",
     "shell.execute_reply": "2025-01-27T14:38:18.045804Z",
     "shell.execute_reply.started": "2025-01-27T14:38:17.596910Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([2, 50, 12507])\n",
      "torch.Size([2, 50, 50])\n"
     ]
    }
   ],
   "source": [
    "model = Sequence2Sequence(src_vocab_size=len(src_word2idx), trg_vocab_size=len(trg_word2idx))\n",
    "#做model的前向传播，看看输出的shape\n",
    "encoder_inputs = torch.randint(0, 100, (2, 50))\n",
    "decoder_inputs = torch.randint(0, 100, (2, 50))\n",
    "attn_mask = torch.randint(0, 2, (2, 50))\n",
    "logits, scores = model(encoder_inputs=encoder_inputs, decoder_inputs=decoder_inputs, attn_mask=attn_mask)\n",
    "print(logits.shape)\n",
    "print(scores.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "db1b6485db0bd5b9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:45.191754Z",
     "start_time": "2025-01-27T13:52:45.186260Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:18.049829Z",
     "iopub.status.busy": "2025-01-27T14:38:18.049514Z",
     "iopub.status.idle": "2025-01-27T14:38:18.053710Z",
     "shell.execute_reply": "2025-01-27T14:38:18.053093Z",
     "shell.execute_reply.started": "2025-01-27T14:38:18.049803Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The model has 35,224,540 trainable parameters\n"
     ]
    }
   ],
   "source": [
    "#帮我计算一下model的总参数量\n",
    "def count_parameters(model):\n",
    "    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "\n",
    "\n",
    "print(f\"The model has {count_parameters(model):,} trainable parameters\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d025b8a73c1b5776",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:45.612387Z",
     "start_time": "2025-01-27T13:52:45.192749Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:18.054497Z",
     "iopub.status.busy": "2025-01-27T14:38:18.054298Z",
     "iopub.status.idle": "2025-01-27T14:38:18.581906Z",
     "shell.execute_reply": "2025-01-27T14:38:18.581332Z",
     "shell.execute_reply.started": "2025-01-27T14:38:18.054476Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The model has 73,010,140 trainable parameters\n"
     ]
    }
   ],
   "source": [
    "#帮我初始化一个4层的GRU的model\n",
    "model = Sequence2Sequence(src_vocab_size=len(src_word2idx), trg_vocab_size=len(trg_word2idx), encoder_num_layers=4,\n",
    "                          decoder_num_layers=4)\n",
    "print(f\"The model has {count_parameters(model):,} trainable parameters\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39a17b7e22ff2c1d",
   "metadata": {},
   "source": [
    "# 训练"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a633e80b6643c86",
   "metadata": {},
   "source": [
    "## 损失函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "fff18a09726ce0bb",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:45.619581Z",
     "start_time": "2025-01-27T13:52:45.613388Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:18.583045Z",
     "iopub.status.busy": "2025-01-27T14:38:18.582585Z",
     "iopub.status.idle": "2025-01-27T14:38:18.587897Z",
     "shell.execute_reply": "2025-01-27T14:38:18.587239Z",
     "shell.execute_reply.started": "2025-01-27T14:38:18.583020Z"
    }
   },
   "outputs": [],
   "source": [
    "# 用来算一批数据的损失\n",
    "def cross_entropy_with_paddings(logits, labels, padding_mask=None):\n",
    "    # logits.shape = [batch size, sequence length, num of classes]\n",
    "    # labels.shape = [batch size, sequence length]\n",
    "    # padding_mask.shape = [batch size, sequence length]\n",
    "\n",
    "    batch_size, seq_len, num_classes = logits.shape\n",
    "    # F.cross_entropy 是 PyTorch 提供的函数，用于计算交叉熵损失\n",
    "    loss = F.cross_entropy(logits.reshape(batch_size * seq_len, num_classes), labels.reshape(-1),\n",
    "                           reduction='none')  # reduction='none'表示不对batch求平均\n",
    "    # loss.shape = [batch size * sequence length]\n",
    "    if padding_mask is None:  # 如果没有padding_mask，就直接求平均\n",
    "        loss = loss.mean()\n",
    "    else:\n",
    "        # 如果提供了 padding_mask，则将padding填充部分的损失去除后计算有效损失的均值。\n",
    "        # 首先，通过将 padding_mask reshape 成一维张量，并取 1 减去得到填充掩码。\n",
    "        # 这样填充部分的掩码值变为 1，非填充部分变为 0。\n",
    "        # 将损失张量与填充掩码相乘，这样填充部分的损失就会变为 0。\n",
    "        # 然后，计算非填充部分的损失和（sum）以及非填充部分的掩码数量（sum）作为有效损失的均值计算。\n",
    "        # (因为上面我们设计的mask的token是0，所以这里是1-padding_mask)\n",
    "\n",
    "        # 将padding_mask reshape成一维张量，mask部分为0，非mask部分为1\n",
    "        padding_mask = 1 - padding_mask.reshape(-1)\n",
    "        loss = torch.mul(loss, padding_mask).sum() / padding_mask.sum()\n",
    "\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "23808a399cd7ed2c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:45.626307Z",
     "start_time": "2025-01-27T13:52:45.620574Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:18.589020Z",
     "iopub.status.busy": "2025-01-27T14:38:18.588649Z",
     "iopub.status.idle": "2025-01-27T14:38:18.594689Z",
     "shell.execute_reply": "2025-01-27T14:38:18.594091Z",
     "shell.execute_reply.started": "2025-01-27T14:38:18.588995Z"
    }
   },
   "outputs": [],
   "source": [
    "class SaveCheckpointsCallback:\n",
    "    def __init__(self, save_dir, save_step=5000, save_best_only=True):\n",
    "        self.save_dir = save_dir  # 保存路径\n",
    "        self.save_step = save_step  # 保存步数\n",
    "        self.save_best_only = save_best_only  # 是否只保存最好的模型\n",
    "        self.best_metric = -np.inf  # 最好的指标，指标不可能为负数，所以初始化为-1\n",
    "        # 创建保存路径\n",
    "        if not os.path.exists(self.save_dir):  # 如果不存在保存路径，则创建\n",
    "            os.makedirs(self.save_dir)\n",
    "\n",
    "    # 对象被调用时：当你将对象像函数一样调用时，Python 会自动调用 __call__ 方法。\n",
    "    # state_dict() 返回模型参数的字典，包括模型参数和优化器参数\n",
    "    # metric 是指标，可以是验证集的准确率，也可以是其他指标\n",
    "    def __call__(self, step, state_dict, metric=None):\n",
    "        if step % self.save_step > 0:\n",
    "            return  # 不是保存步数，则直接返回\n",
    "\n",
    "        if self.save_best_only:\n",
    "            assert metric is not None  # 必须传入metric\n",
    "            if metric >= self.best_metric:  # 如果当前指标大于最好的指标\n",
    "                # save checkpoint\n",
    "                # 保存最好的模型，覆盖之前的模型，不保存step，只保存state_dict，即模型参数，不保存优化器参数\n",
    "                torch.save(state_dict, os.path.join(self.save_dir, \"seq2seq_layer_4.ckpt\"))\n",
    "                self.best_metric = metric  # 更新最好的指标\n",
    "        else:\n",
    "            # 保存模型\n",
    "            torch.save(state_dict, os.path.join(self.save_dir, f\"{step}.ckpt\"))\n",
    "            # 保存每个step的模型，不覆盖之前的模型，保存step，保存state_dict，即模型参数，不保存优化器参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "1ac803ec5a65f989",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:45.634181Z",
     "start_time": "2025-01-27T13:52:45.628299Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:18.595814Z",
     "iopub.status.busy": "2025-01-27T14:38:18.595365Z",
     "iopub.status.idle": "2025-01-27T14:38:18.600119Z",
     "shell.execute_reply": "2025-01-27T14:38:18.599601Z",
     "shell.execute_reply.started": "2025-01-27T14:38:18.595790Z"
    }
   },
   "outputs": [],
   "source": [
    "class EarlyStopCallback:\n",
    "    def __init__(self, patience=5, min_delta=0.01):\n",
    "        self.patience = patience  # 多少个step没有提升就停止训练\n",
    "        self.min_delta = min_delta  # 最小的提升幅度\n",
    "        self.best_metric = -np.inf  # 记录的最好的指标\n",
    "        self.counter = 0  # 计数器，记录连续多少个step没有提升\n",
    "\n",
    "    def __call__(self, metric):\n",
    "        if metric >= self.best_metric + self.min_delta:  # 如果指标提升了\n",
    "            self.best_metric = metric  # 更新最好的指标\n",
    "            self.counter = 0  # 计数器清零\n",
    "        else:\n",
    "            self.counter += 1  # 计数器加一\n",
    "\n",
    "    @property  # 使用@property装饰器，使得 对象.early_stop可以调用，不需要()\n",
    "    def early_stop(self):\n",
    "        # 如果计数器大于等于patience，则返回True，停止训练\n",
    "        return self.counter >= self.patience"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8403c3b46f8ed34",
   "metadata": {},
   "source": [
    "## training & valuating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "d782c394b39d9d03",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:45.640534Z",
     "start_time": "2025-01-27T13:52:45.636175Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:18.601204Z",
     "iopub.status.busy": "2025-01-27T14:38:18.600811Z",
     "iopub.status.idle": "2025-01-27T14:38:18.605275Z",
     "shell.execute_reply": "2025-01-27T14:38:18.604795Z",
     "shell.execute_reply.started": "2025-01-27T14:38:18.601182Z"
    }
   },
   "outputs": [],
   "source": [
    "# 用来算整个test_loader的损失\n",
    "@torch.no_grad()  # 装饰器，禁止梯度计算\n",
    "def evaluating(model, dataloader, loss_fct):\n",
    "    loss_list = []\n",
    "    for batch in dataloader:\n",
    "        encoder_inputs = batch[\"encoder_inputs\"]\n",
    "        encoder_inputs_mask = batch[\"encoder_inputs_mask\"]\n",
    "        decoder_inputs = batch[\"decoder_inputs\"]\n",
    "        decoder_labels = batch[\"decoder_labels\"]\n",
    "        decoder_labels_mask = batch[\"decoder_labels_mask\"]\n",
    "\n",
    "        # 前向计算\n",
    "        logits, scores = model(\n",
    "            encoder_inputs=encoder_inputs,\n",
    "            decoder_inputs=decoder_inputs,\n",
    "            attn_mask=encoder_inputs_mask\n",
    "        )  # model就是seq2seq模型\n",
    "\n",
    "        # 验证集损失\n",
    "        loss = loss_fct(logits, decoder_labels, padding_mask=decoder_labels_mask)\n",
    "        loss_list.append(loss.cpu().item())\n",
    "\n",
    "    return np.mean(loss_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "f72acf09bc531de1",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:45.650208Z",
     "start_time": "2025-01-27T13:52:45.642088Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:18.606401Z",
     "iopub.status.busy": "2025-01-27T14:38:18.605981Z",
     "iopub.status.idle": "2025-01-27T14:38:18.614673Z",
     "shell.execute_reply": "2025-01-27T14:38:18.614106Z",
     "shell.execute_reply.started": "2025-01-27T14:38:18.606380Z"
    }
   },
   "outputs": [],
   "source": [
    "def training(model,\n",
    "             train_loader,\n",
    "             val_loader,\n",
    "             epoch,\n",
    "             loss_fct,\n",
    "             optimizer,\n",
    "             save_ckpt_callback=None,\n",
    "             early_stop_callback=None,\n",
    "             eval_step=500,\n",
    "             ):\n",
    "    record_dict = {\n",
    "        \"train\": [],\n",
    "        \"val\": []\n",
    "    }\n",
    "\n",
    "    global_step = 1  # 全局步数\n",
    "    model.train()  # 训练模式\n",
    "    with tqdm(total=epoch * len(train_loader)) as pbar:\n",
    "        for epoch_id in range(epoch):\n",
    "            for batch in train_loader:\n",
    "                encoder_inputs = batch[\"encoder_inputs\"]\n",
    "                encoder_inputs_mask = batch[\"encoder_inputs_mask\"]\n",
    "                decoder_inputs = batch[\"decoder_inputs\"]\n",
    "                decoder_labels = batch[\"decoder_labels\"]\n",
    "                decoder_labels_mask = batch[\"decoder_labels_mask\"]\n",
    "\n",
    "                # 前向传播\n",
    "                logits, scores = model(\n",
    "                    encoder_inputs=encoder_inputs,\n",
    "                    decoder_inputs=decoder_inputs,\n",
    "                    attn_mask=encoder_inputs_mask\n",
    "                )\n",
    "                loss = loss_fct(logits, decoder_labels, padding_mask=decoder_labels_mask)\n",
    "\n",
    "                # 反向传播\n",
    "                optimizer.zero_grad()  # 梯度清零\n",
    "                loss.backward()  # 反向传播\n",
    "                optimizer.step()  # 优化器更新参数\n",
    "\n",
    "                loss = loss.cpu().item()\n",
    "\n",
    "                record_dict[\"train\"].append({\n",
    "                    \"loss\": loss,\n",
    "                    \"step\": global_step\n",
    "                })\n",
    "\n",
    "                # 评估\n",
    "                if global_step % eval_step == 0:\n",
    "                    model.eval()  # 评估模式\n",
    "                    # 验证集损失和准确率\n",
    "                    val_loss = evaluating(model, val_loader, loss_fct)\n",
    "                    record_dict[\"val\"].append({\n",
    "                        \"loss\": val_loss,\n",
    "                        \"step\": global_step\n",
    "                    })\n",
    "                    model.train()  # 训练模式\n",
    "\n",
    "                    # 2. 保存模型权重 save model checkpoint\n",
    "                    if save_ckpt_callback is not None:\n",
    "                        # model.state_dict() 返回模型参数的字典，包括模型参数和优化器参数\n",
    "                        save_ckpt_callback(global_step, model.state_dict(), -val_loss)\n",
    "                        # 保存最好的模型，覆盖之前的模型，保存step，保存state_dict,通过metric判断是否保存最好的模型\n",
    "\n",
    "                    # 3. 早停 early stopping\n",
    "                    if early_stop_callback is not None:\n",
    "                        # 验证集准确率不再提升，则停止训练\n",
    "                        early_stop_callback(-val_loss)\n",
    "                        # 验证集准确率不再提升，则停止训练\n",
    "                        if early_stop_callback.early_stop:\n",
    "                            print(f\"Early stop at epoch {epoch_id} / global_step {global_step}\")\n",
    "                            return record_dict  # 早停，返回记录字典 record_dict\n",
    "\n",
    "                # 更新进度条和全局步数\n",
    "                pbar.update(1)  # 更新进度条\n",
    "                global_step += 1  # 全局步数加一\n",
    "                pbar.set_postfix({\"epoch\": epoch_id})\n",
    "\n",
    "    return record_dict  # 训练结束，返回记录字典 record_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "e53ee784c3c7e957",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:53:27.384491Z",
     "start_time": "2025-01-27T13:52:45.651204Z"
    },
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T14:38:18.615788Z",
     "iopub.status.busy": "2025-01-27T14:38:18.615427Z",
     "iopub.status.idle": "2025-01-27T15:01:09.117078Z",
     "shell.execute_reply": "2025-01-27T15:01:09.116498Z",
     "shell.execute_reply.started": "2025-01-27T14:38:18.615753Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  7%|▋         | 12499/167200 [22:48<4:42:22,  9.13it/s, epoch=7] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stop at epoch 7 / global_step 12500\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "batch_size = 64\n",
    "train_loader = DataLoader(train_ds, batch_size=batch_size, shuffle=True, collate_fn=collate_fn)\n",
    "test_loader = DataLoader(test_ds, batch_size=batch_size, shuffle=False, collate_fn=collate_fn)\n",
    "\n",
    "epoch = 100\n",
    "\n",
    "model = Sequence2Sequence(src_vocab_size=len(src_word2idx), trg_vocab_size=len(trg_word2idx),encoder_num_layers=4,decoder_num_layers=4)\n",
    "\n",
    "model.to(device)\n",
    "\n",
    "# 1. 定义损失函数 采用交叉熵损失\n",
    "loss_fct = cross_entropy_with_paddings\n",
    "\n",
    "# 2. 定义优化器\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
    "\n",
    "# 3.save model checkpoint\n",
    "if not os.path.exists(\"checkpoints\"):\n",
    "    os.makedirs(\"checkpoints\")\n",
    "save_ckpt_callback = SaveCheckpointsCallback(save_dir=\"checkpoints\", save_step=500, save_best_only=True)\n",
    "\n",
    "# 4. early stopping\n",
    "early_stop_callback = EarlyStopCallback(patience=5, min_delta=0.01)\n",
    "\n",
    "# 训练过程\n",
    "record_dict = training(\n",
    "    model,\n",
    "    train_loader,\n",
    "    test_loader,\n",
    "    epoch,\n",
    "    loss_fct,\n",
    "    optimizer,\n",
    "    save_ckpt_callback=save_ckpt_callback,\n",
    "    early_stop_callback=early_stop_callback,\n",
    "    eval_step=500\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "43030e8b25455e24",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:53:27.386486Z",
     "start_time": "2025-01-27T13:53:27.385486Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:01:09.118380Z",
     "iopub.status.busy": "2025-01-27T15:01:09.117913Z",
     "iopub.status.idle": "2025-01-27T15:01:09.228943Z",
     "shell.execute_reply": "2025-01-27T15:01:09.228189Z",
     "shell.execute_reply.started": "2025-01-27T15:01:09.118357Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhYAAAGdCAYAAABO2DpVAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAVPBJREFUeJzt3XdcU1fjBvDnJoQAygYRFERw4N4DZ1tRa+lutVXb2tqtb6v1rR2/1tYOX+14u1u7bfu2aqedWsVR9957ozgAESHMEJLz+yMkJGZAMHDh8nw/Hz+Se09uTg5KHs49QxJCCBARERF5gUruChAREZFyMFgQERGR1zBYEBERkdcwWBAREZHXMFgQERGR1zBYEBERkdcwWBAREZHXMFgQERGR1/jU9QuaTCacO3cOgYGBkCSprl+eiIiIakAIgYKCAsTExEClct0vUefB4ty5c4iNja3rlyUiIiIvyMjIQMuWLV2er/NgERgYCMBcsaCgIK9d12AwYNmyZRgxYgQ0Go3XrqsUbB/32D5VYxu5x/Zxj+3jXkNoH51Oh9jYWOvnuCt1Hiwstz+CgoK8HiwCAgIQFBRUb78pcmL7uMf2qRrbyD22j3tsH/caUvtUNYyBgzeJiIjIaxgsiIiIyGsYLIiIiMhrGCyIiIjIaxgsiIiIyGsYLIiIiMhrGCyIiIjIaxgsiIiIyGsYLIiIiMhrGCyIiIjIaxgsiIiIyGsYLIiIiMhrFBMs3llxDD+fVCFLVyp3VYiIiBqtOt/dtLb8sO0MLhSqkFtkQMtwuWtDRETUOCmmx0KlMm/jahJC5poQERE1XooJFuqK/eGNJgYLIiIiuSgmWFh6LIzssSAiIpKNYoKFJHcFiIiISDnBgoiIiOSnvGDBOyFERESyUUywkHgvhIiISHaKCRZEREQkPwYLIiIi8hrFBQsOsSAiIpKPYoKFxAmnREREslNMsCAiIiL5KS5YCK68SUREJBvFBAtONyUiIpKfYoIFERERyY/BgoiIiLxGccGCIyyIiIjko5hgwSEWRERE8lNMsCAiIiL5eRwsCgoKMHXqVLRq1Qr+/v4YMGAAtm7dWht1qxHONiUiIpKPx8HigQceQFpaGv73v/9h7969GDFiBFJSUnD27NnaqF+1cbopERGR/DwKFiUlJfj555/x+uuvY8iQIWjTpg1mzpyJNm3aYO7cubVVRyIiImogfDwpXF5eDqPRCD8/P7vj/v7+WLdundPn6PV66PV662OdTgcAMBgMMBgMntbXJcstkPLycq9eVyksbcK2cY7tUzW2kXtsH/fYPu41hPapbt0k4eEa2AMGDICvry/mz5+PqKgoLFiwABMmTECbNm1w+PBhh/IzZ87ESy+95HB8/vz5CAgI8OSl3Zq1U43sUgmPdSpHmyCvXZaIiIgAFBcXY9y4ccjPz0dQkOsPWo+DxfHjxzFx4kSsWbMGarUaPXv2RLt27bB9+3YcPHjQobyzHovY2Fjk5OS4rZinRr67DidyivHNhB5IbhPptesqhcFgQFpaGoYPHw6NRiN3deodtk/V2EbusX3cY/u41xDaR6fTISIiospg4dGtEABITEzE6tWrUVRUBJ1Oh+joaNxxxx1ISEhwWl6r1UKr1Toc12g0Xm488+hNtY+63n5T6gPvt7uysH2qxjZyj+3jHtvHvfrcPtWtV43XsWjSpAmio6Nx6dIlLF26FDfddFNNL+VVnG5KREQkH497LJYuXQohBNq3b49jx45h+vTpSEpKwn333Vcb9as2TjclIiKSn8c9Fvn5+Zg8eTKSkpJwzz33YNCgQVi6dGm97bohIiKiuuNxj8WYMWMwZsyY2qgLERERNXCK2SuEd0KIiIjkp5hgQURERPJjsCAiIiKvUVyw4HRTIiIi+SgmWHC6KRERkfwUEyyIiIhIfooLFgK8F0JERCQXxQQLiRNOiYiIZKeYYEFERETyY7AgIiIir1FcsOB0UyIiIvkoJlhwuikREZH8FBMsiIiISH6KCxa8E0JERCQfxQQL3gkhIiKSn2KCBREREclPccGCs0KIiIjko5xgwWkhREREslNOsCAiIiLZMVgQERGR1yguWHB3UyIiIvkoJlhwhAUREZH8FBMsiIiISH7KCxa8E0JERCQbxQQLzjYlIiKSn2KCBREREcmPwYKIiIi8RnHBgkMsiIiI5KOYYMExFkRERPJTTLAgIiIi+SkuWAhub0pERCQbxQQLiWtvEhERyU4xwYKIiIjkx2BBREREXqO4YMERFkRERPJRTLDgdFMiIiL5KSZYEBERkfwUFyw425SIiEg+igkWvBNCREQkP8UECyIiIpKf4oIF74QQERHJRznBgvdCiIiIZKecYEFERESyY7AgIiIir/EoWBiNRsyYMQOtW7eGv78/EhMT8corr9SrHUXrU12IiIgaGx9PCr/22muYO3cuvv76a3Tq1Anbtm3Dfffdh+DgYDz++OO1Vcdq4e6mRERE8vMoWGzYsAE33XQTUlNTAQDx8fFYsGABtmzZUiuVIyIioobFo1shAwYMwIoVK3DkyBEAwO7du7Fu3TqMGjWqVipXI7wTQkREJBuPeiyeeeYZ6HQ6JCUlQa1Ww2g0YtasWRg/frzL5+j1euj1eutjnU4HADAYDDAYDDWsthMVYyvKjUbvXlchLG3CtnGO7VM1tpF7bB/32D7uNYT2qW7dJOHBaMeFCxdi+vTpeOONN9CpUyfs2rULU6dOxVtvvYUJEyY4fc7MmTPx0ksvORyfP38+AgICqvvSVXp7rxrphRIeaG9ElzB2WxAREXlTcXExxo0bh/z8fAQFBbks51GwiI2NxTPPPIPJkydbj7366qv49ttvcejQIafPcdZjERsbi5ycHLcV89TtH2/C7rM6fHBHF4zsHO216yqFwWBAWloahg8fDo1GI3d16h22T9XYRu6xfdxj+7jXENpHp9MhIiKiymDh0a2Q4uJiqFT2wzLUajVMJpPL52i1Wmi1WofjGo3Gq40nqSRrferrN6U+8Ha7Kw3bp2psI/fYPu6xfdyrz+1T3Xp5FCxuuOEGzJo1C3FxcejUqRN27tyJt956CxMnTqxRJb2Jk02JiIjk51GweP/99zFjxgxMmjQJ2dnZiImJwcMPP4wXXnihtupHREREDYhHwSIwMBDvvPMO3nnnnVqqzpXjwptERETyUcxeIZLEmyFERERyU0ywICIiIvkxWBAREZHXKC5YCK7pTUREJBvFBAuOsCAiIpKfYoIFERERyU9xwYLTTYmIiOSjmGDB2aZERETyU0ywICIiIvkpLljwTggREZF8FBcsiIiISD4MFkREROQ1DBZERETkNYoLFoLzTYmIiGSjmGDB3U2JiIjkp5hgQURERPJTTLBgfwUREZH8FBMsiIiISH6KCxYcu0lERCQfxQQLjt0kIiKSn2KCBREREclPccGCd0KIiIjko5hgwTshRERE8lNMsLDgyptERETyUUyw4MqbRERE8lNMsCAiIiL5KS5Y8EYIERGRfBQTLHgjhIiISH6KCRZEREQkP8UFC04KISIiko9yggXvhRAREclOOcGiAjssiIiI5KOYYCGxy4KIiEh2igkWREREJD/lBQuO3iQiIpKNYoIFV/QmIiKSn2KCBREREclPccGCN0KIiIjko5hgwTshRERE8lNMsLDg2E0iIiL5KCZYcPAmERGR/BQTLIiIiEh+igsWgsM3iYiIZKOYYMElvYmIiOSnmGBBRERE8vMoWMTHx0OSJIc/kydPrq36eYyzQoiIiOTj40nhrVu3wmg0Wh/v27cPw4cPx+jRo71eMY/xTggREZHsPAoWkZGRdo/nzJmDxMREDB061KuVuhLssCAiIpKPR8HCVllZGb799ltMmzYNkptFJPR6PfR6vfWxTqcDABgMBhgMhpq+vKOKeyBGo9G711UIS5uwbZxj+1SNbeQe28c9to97DaF9qls3SYiajUr44YcfMG7cOJw+fRoxMTEuy82cORMvvfSSw/H58+cjICCgJi/t1CcHVTiQp8LYRCP6N2O/BRERkTcVFxdj3LhxyM/PR1BQkMtyNQ4WI0eOhK+vL/744w+35Zz1WMTGxiInJ8dtxTz1wDfbsfroRbxyQxLu7BvntesqhcFgQFpaGoYPHw6NRiN3deodtk/V2EbusX3cY/u41xDaR6fTISIiospgUaNbIadOncLy5cvxyy+/VFlWq9VCq9U6HNdoNF5tPJXKfDtGrVbX229KfeDtdlcatk/V2EbusX3cY/u4V5/bp7r1qtE6FvPmzUOzZs2Qmppak6cTERGRQnkcLEwmE+bNm4cJEybAx6fGYz9rEcdXEBERycXjYLF8+XKcPn0aEydOrI361BiX9CYiIpKfx10OI0aMQA3He9aJelw1IiIixVPMXiFultIgIiKiOqKYYEFERETyU1yw4J0QIiIi+SgmWPBOCBERkfwUEyyIiIhIfooLFpwVQkREJB/FBAt3O6wSERFR3VBMsLAQHL5JREQkG8UFCyIiIpIPgwURERF5jeKCBQdvEhERyUcxwYJjN4mIiOSnmGBBRERE8lNcsOCdECIiIvkoJljwTggREZH8FBMsrDh6k4iISDaKCRZceZOIiEh+igkWREREJD/FBQveCCEiIpKPYoIFb4QQERHJTzHBgoiIiOSnuGDBSSFERETyUUyw4KQQIiIi+SkmWFiww4KIiEg+igkWEodvEhERyU4xwYKIiIjkp7hgITh6k4iISDbKCRa8E0JERCQ75QQLIiIikp3iggVvhBAREclHMcGCd0KIiIjkp5hgYcGxm0RERPJRXLAgIiIi+SgmWPy5NxMA8N+0ozLXhIiIqPFSTLCw0Jeb5K4CERFRo6W4YEFERETyYbAgIiIir2GwICIiIq9hsCAiIiKvYbAgIiIir2GwICIiIq9hsCAiIiKvYbAgIiIir2GwICIiIq/xOFicPXsWd911F8LDw+Hv748uXbpg27ZttVE3IiIiamB8PCl86dIlDBw4EFdffTWWLFmCyMhIHD16FKGhobVVPyIiImpAPAoWr732GmJjYzFv3jzrsdatW3u9UkRERNQweRQsfv/9d4wcORKjR4/G6tWr0aJFC0yaNAkPPvigy+fo9Xro9XrrY51OBwAwGAwwGAw1rLZ7tXXdhszSJmwb59g+VWMbucf2cY/t415DaJ/q1k0SQojqXtTPzw8AMG3aNIwePRpbt27FlClT8PHHH2PChAlOnzNz5ky89NJLDsfnz5+PgICA6r50laZsrMxI7yaXe+26REREBBQXF2PcuHHIz89HUFCQy3IeBQtfX1/07t0bGzZssB57/PHHsXXrVmzcuNHpc5z1WMTGxiInJ8dtxTzVdsYy69dHXxnhtesqhcFgQFpaGoYPHw6NRiN3deodtk/V2EbusX3cY/u41xDaR6fTISIiospg4dGtkOjoaHTs2NHuWIcOHfDzzz+7fI5Wq4VWq3U4rtFoaq3x6us3pT6ozXZXArZP1dhG7rF93GP7uFef26e69fJouunAgQNx+PBhu2NHjhxBq1atPLkMERERKZRHweKJJ57Apk2b8J///AfHjh3D/Pnz8emnn2Ly5Mm1VT8iIiJqQDwKFn369MGiRYuwYMECdO7cGa+88greeecdjB8/vrbqR0RERA2IR2MsAOD666/H9ddfXxt1ISIiogaOe4UQERGR1ygyWBiMJrmrQERE1CgpJlgE+KqtX5uqvzQHEREReZFigoXtOl8SJBlrQkRE1HgpJli0iwq0fi3AHgsiIiI5KCZY3DegcpEu3gkhIiKSh2KChdan8q0wWBAREclDMcHCVonBKHcViIiIGiVFBovP1p6QuwpERESNkiKDxa87z8pdBSIiokZJMcFCsplhej6/VL6KEBERNWKKCRZEREQkPwYLIiIi8hrFBAtJ4mqbREREclNMsCAiIiL5KSZYqNhhQUREJDvFBIvkhHC5q0BERNToKSZY2C7pTURERPLgpzERERF5TaMMFvpy7iVCRERUGxpdsHhr2WG0f/5vbD+VK3dViIiIFKfRBYv3Vh4DALz850GZa0JERKQ8jS5YWAkhdw2IiIgUp/EGCyIiIvI6BgsiIiLyGgYLIiIi8ppGGyw4woKIiMj7FBssdmfkuT1vMDJaEBEReZtig8VNH653e/7geV0d1YSIiKjxUGywICIiorrHYEFERERew2BBREREXsNgQURERF7DYEFEREReo+hgIbgfCBERUZ1SdLAwmhgsiIiI6pKig8Udn26SuwpERESNiqKDxfZTl1BqMMpdDSIiokZD0cECAD5cdUzuKhARETUaig8WO0/nyV0FIiKiRkPxwWLdsRzsOH1J7moQERE1CooKFt3DTE6P3/kJB3ESERHVBUUFixtbOQ8WZUbnx4mIiMi7FBUs1JLcNSAiImrcPAoWM2fOhCRJdn+SkpJqq24e83Q5LC6gRURE5F0+nj6hU6dOWL58eeUFfDy+RL2xKyMPvVqFyl0NIiIixfA4Ffj4+KB58+a1UZcrFuLrWfnTuUUMFkRERF7kcbA4evQoYmJi4Ofnh+TkZMyePRtxcXEuy+v1euj1eutjnU4HADAYDDAYDDWosnMGgwGSBLwzuhOm/rjf6fnLPf3zXlzfOcprdajPLO/fm22uJGyfqrGN3GP7uMf2ca8htE916yYJD7YAXbJkCQoLC9G+fXucP38eL730Es6ePYt9+/YhMDDQ6XNmzpyJl156yeH4/PnzERAQUN2XrrbdFyV8eUTtcPzd5HIAwJSNPk6PExERkWvFxcUYN24c8vPzERQU5LKcR8Hicnl5eWjVqhXeeust3H///U7LOOuxiI2NRU5OjtuKecpgMCAtLQ1tegxE6kebHc4ffWUEAKDtjGVOjyudpX2GDx8OjUYjd3XqHbZP1dhG7rF93GP7uNcQ2ken0yEiIqLKYHFFIy9DQkLQrl07HDvmej8OrVYLrVbrcFyj0dRK47WLDnZ63NVr1ddvYG2prXZXCrZP1dhG7rF93GP7uFef26e69bqidSwKCwtx/PhxREdHX8lliIiISCE8ChZPPvkkVq9ejfT0dGzYsAG33HIL1Go1xo4dW1v185oHvt4KE9etICIiqlUeBYszZ85g7NixaN++PcaMGYPw8HBs2rQJkZGRtVW/Gknp4DjTY/nBbLy+9LAMtSEiImo8PBpjsXDhwtqqh1dd1T4Syw9mORz/ePVxh2PHLxQiMbJpXVSLiIhI8RS1V4hFUnPnU1+dGf+Z4wwSIiIiqhlFBotusSHVLpupK0WpwVh7lSEiImpEFBksPJU8e4XcVSAiIlIERQYLT3dPv1Rcf5dQJSIiakga7takbkiSp9HC7PO1J7AzIw/9E8Jxd/9WXq4VERGR8ikyWNTEoUwdXv3rIADgrz3nMahNBFpHNJG5VkRERA0Lb4VUWLznvN3jvOIy71SGiIioEVFksFCpPI8W76103O9kz5k86MvNM0Yycovx9E97cCy74IrrR0REpFSKDBbe8MW6k7jxg/WY/N0OAMDEr7bi+20ZuP3jjTLXjIiIqP5isHDhz4pbI8sPZgMAjmYXAgDyOIOEiIjIJQaLGnh3+VG7x+VGEwr15V5/nYzcYhi5cRoRETUgDBY18PbyI3aPR7yzBp1fXIrcIu8N+Fy2PxODX1+F+7/e6rVrEhER1TYGixoSorIn4cSFIgBA6ntr8eSPu+3O1dSX608CAP45fOGKr0VERFRXFBssbunRolav3/rZxfh151m7Y+fzS/HT9jM4mWMOGoX6chSU1mxMhheyCRERUZ1TbLB4+47uSOnQzCvXWnvUea/B1O93OT3+9M97sPboBXR+cSm6zFyGsnKTx6/JXEFERA2RYoMFAAxIjPDKde7+YotH5bemX7J7zsUivVfqQUREVN8pOljck1z7+33kV2P66ZXe1kg7kHVlFyAiIqojig4WPmoVbuoeU6uv0e3lZbVzYZsw8uA322rnNYiIiLxM0cECAEZ1jpa7CjifX1qtcj9tP4Nnft4Do0lAcJQFERE1QIrf3bSGO6h71W1zNyB9Tqr1cUZuMRbtPIu4sACkdIxCU6352/Dkj7sBAMmJ4bLUk4iI6EopPljUF5eKyhDaxBcAcN17a1FQal6ps2/rMNzVvxX6J4RZy7paNvz9FUdxJLsQ797RvUYbrREREdU2xQeL+vLx2+OVNKfHt5zMxZaTuXbHDmcVOB3w+d8084qfY/vEYkAb78x4ISIi8ibFj7FIiGwidxU8Nn/zaYdjc/85bv26tGIrdyIiovpG8cGiTbNAzLuvj8Pxk7Ovk6E2Nffa34fkrgIREVGVFB8sAODq9s0c1rSQ6sOoTjf0NVitk4iISG6NIlgAwNSUdugZF4KUDlFY+9TVLssNV23DjaoNkHtR7b1n812eW7w3E+8uP+qw2VlN9yUhIiLyFsUP3rQIa+KLXyYNdF8GOszRfIZwqQDXGTfjecNE5CC4jmpYfT9tPwMA6J8Qhn4J5qmpn605gVmLD+Lfw9shv8SAe5LjsexAJnrEhaBXqzB3l6sVS/aex9b0S3gutQPU1ZjBkl1QijOXStAzLrQOakdERLWl0QSL6tAhAF+Xj8RjPotwrXor+qkO4kXDvfjdlIz6M7+kUk5hmfXrWYsPAqicOfL5upPWc7ZraFyJbF0pQgJ84etTdUfXo9/tAAB0iw3GTd2r3mm276wVAIBFkwagB8MFEVGD1WhuhVRHOXzwnvFW3Fj2KvaZ4hEqFeI93w/wieZtRCJP7uo5KNKX49tNp7AtPbfqwgDOFwNvLjuKM5eKXZZZcTALL/2xHwaj/RiPQ5k69P3PCtz4wTq747sz8vDd5lMOt2UsLhSYN2A7cE6HSd9tx/ELhW7ruOlE9d4LERHVT426xyK1SzT+2nve4fhB0Qo3l72MR9W/4zGfRRip3oa+qkN40TABv5sGoL70Xjz1855qlZu6cCdu6R6NObt9AJzEJ2tP4vd/DUTXliEOZe//2rwvSZtmTTG+X+WA1992nQMAHMossCt/04frAQBRgX5I6Rjlsg63fLQe+nITdmfkY/0z17gs99rfh/DI0IR6P7iWiIica9Q9Fh+M64G9M0c4PVcOH7xvvBU3ls2y6b34sN72Xrjz665zmPDVdrtjX9rcKrGY+ft+69fPLdqHs3kl1sdVfcwfza7sifh87Qnr1ydzijB14U7rLBfba7qy4/SlKssQEVH91KiDhSRJCPTToGdciMsyh0Qcbi57Gf813I4yocZI9TYs0z6FG1XrIffMkSthrKi6EAJvpR3Bkr3n8dWGdLsyL/5WGTRsOxBKyowwmpy/90tFZXj1r4PWx99tPo1fK3o7qqu4jAuAERE1VI06WFj89MgAHH71Wqx7+mo8OLi1w3kl9V5Y/LHb/GG/9mgO3ltx1DrY0tbp3CKUOPmQ7/DC37jh/XWYvcQ2QJxCudFU7fU3Pl97And/sRmlBsfrS/XkVhMREXmOwQKASiVB66NGy9AAPJfa0WU5Z70XadrpuEm1Dg2x96JIX44snest3Y9kFaLvrOUo1Jfjw1XH7c4dOK/DJ6srb3mcuVSCBVsclyJ35q895/HqXwex9mgOkmb8jR+2ZtTsDRARUb3DYOEh296LvaZ4hEhFeNf3I3yqeQuRaFhjA279aAP2n9O5LVOgL8d/lx2u1vV2ZuRVa5v6yfPte0cuH4RqdDHDpDbkFZdh/bEcmFzc2qkuIQS+WHcSm09c9FLNiIgaJgaLGjok4nBL2ct40zAaZUKNEertSNM+1aB6Lw5nFTiMq3Bm3vqqywDALzvOXlmFKkz4cgsycotxPr/qgZ4AkJFbjEJ9eY1eK/W9dRj/+Wb8dIV1X3EwG6/8eQB3fLrpiq4jl8/XnsAtH62Hjqu3EtEVYrC4AuXwwQfGWxx6Lz5rgL0X3uKt2xqDX1+F5Nkr8e7yo/ht11noXezoejKnCINfX4U+ry6v0etYZqk899sBeNJp8dmaE5j41VaUVYwpSb9Y5LRckb4c209duuIekdr26l8HsfN0Hr5Y6zhbiDwnhMAfu8/hWLb7dVuIlIjBwgssvRdvGMagTKgxvKL3Yox6FZrC9WJUSmRZ+dNb3l5+BFMW7kL75//Gc4v2OpxfdywHAFDiZBCopy64Hm7iYNbig1h5KBu/7jL3dNgOWrUdkHrnp5tw29wN+GFbZeD6fO0JLNufecX1rQ3OBtOS51YczMZjC3Yi5a3VcleFqM4xWHhJOXzwofFm3GDTe/G65jPs1D6M731fxiT1r+gsnYAE7lpaU99tPo3sglLc+tF6zP3HPJj0VE5lT0G50YSM3GJkF5Tin8PZLlcDvVK2t12+Wp+Oc3kldscKSiu/tmwm9/MO8/4uuzLy8OpfB/HQ/+zXFakv6ne/SsOx+0ye3FUgkk2jXnnTledTO+DnHWdx8Lz7gY3OHK7ovXhAvRij1auRqDqPftIh9FMdwlP4ATkiCOtMnbHa2A1rTV3r5SZn9dn/Np7CjtN52HE6D/ckt7LbE+W+r7Zi7dEc62O1SsLx/1zn9TpMWbDT+vWB8zqMfHsN7kpu5eYZgCXjZLuZhUNEpAQMFk48MDgBEwbEo+1zS2r0/HL44GPjjfjYeCNaStkYqtqDoardSFYdQISkw83qDbhZvQEAsN/UCqtN3bDG1BXbTe1g4LfEraU2txDeW3HU7pxtqABgXcTLYDTh7KUSxEc0sZ77ffc5p7dWXMktKsMbSw/jjj6xWHEo2+5cgb4cl3eO5BTq8dRP1VtyvTYYTQITv9qK9s0D8X/Xdaj282qrl+fy11Daku0ZucUoLavcFFBZ747IM/wUc0GjVuGJlHYoLivHJ2tOVP0EF86IZvjOmILvjCnwQTl6SkcxVL0bQ1R70EWVjk6qU+ikOoVJ+B2Fwg8bTR2tQeO0cL33RmN1JKtyMFx1vy93f7EZm07kom98GH54JBkA8LhNr4PFTydVuK/i6+yCUkQ00UJVseX7q38dwC87zrpcq+Pj1ZXrfAgIpPx3DfJLKmdY7DuXX3HOOy4VlSHIX+OwJf3O05fwwcpjuCqpGVYfuYDVRy54GCxqVp+ychNWHspC/4RwBPj64Invd2FgmwiM6xdnV275gSw8+dNuvH1Hd8SFBSC/xICeNdzNtr4EFKNJYPDrqwAAr/eVuTJE9QCDhRtTUtoCqP4HWFXK4YMtogO2lHfAG7gTEcjHINVeDFHvwWDVHkRKOgxX78BwtXmdh3RTFNaYumK9qTMOiDicEZEQHBbjMcuOqVvSczF14U6Xa3ccyVdh/BdbMTWlHcZ9vhkA0Dc+DC/e2NHj0f22oQIASg2OY2sOntdhwpdbkNIxCjNSO8LfV43fdp3FgfM6PHNtkssPzSNZBRjx9hr0iQ/Fj48MsDt3y0fmnrDLe1UsisvK8dee8xjWIQphTXwdztc0+Nzx6UbsPJ2HmGA/PDasLf7aex5/7T3vECwe+Ma8yd1987Zaj2145hrEhPh79HrLD2Rh+k+78dYd3XF1+2Y1rLV3lNkM3C3kbF0rIQSWHchCx+ggxIYFyF0dqkMMFtXwfGoHvPb3IQgBlHtx2mAOgvGraRB+NQ2CBBM6SqfMt03Uu9FTOop4VRbiVWm4B2kAgELhhyOiJQ6ZYnFYxOGwiMUhUyzyEOi1OinN5bMcqtq3ZEv6Jbxjc4tlS3ouUt9b5+YZjjJync8EuvadNUiIrLwd89iCncgu0GP+5tM4mlWAHx8ZgCkLdwEABiRGYGi7SIdrCCEw4u01AICt6fZTmvdVDBS9XLnRBB+1CkaTwIxf9+PnHWfQuUUQfnl0IHx97IOqsNlD5nBWARIimjqUudzxC4XYeToPAHAuv9TjGS/pOUUeBYsifbldQDk5+zrzKq7RgWgW6OfRa1eXySSsvVeXExzy6tTf+zKtWwWkz0mVuTa1SwgBfbkJfhq13FWpFxgsquGBwQm4d0A8VJKEMqMJSTP+9vprCKiwX7TGfmNrfGS8CU1RjGTVAQxV7UZP1TEkSmfRVCpFT+kYeqqO2T03S4TgsCkWh0Rcxd+xOCZaQA/H30gbm0e/9Xz2hYvPj2q7be5Gp8cPZRbYbTtv2wuyNf0Sbp+7wfo4t0jv9Bq6UtcLgR2/4LxXpc1zS9C7VSj2ns23Tovdd1aHds8vwQvXd8TEQZX741g+JBduzcCzv+zF1e0j8fJNnbH2aA5u7Gq+Nbf2aA7eXXUCvVuFYvrI9jhi854AYNXhCy7r6IynH8sv/3HA7vGSfZmYVPEBdmzWKPioVdidkYcyowl94sOqfd1DmTr8suMsJl/VBsEBGuvxXRl5uOvzzXjq2va4Jznesf42b8DayVQPbtFYFJQacNOH6zEsqZnbLQu8bfPJ3Dp7Lbk9vnAX/th9Dv88eZXdWK7G6oqCxZw5c/Dss89iypQpeOedd7xUpfrJR23+rc1PVb1E2jc+DFvSa/4fqxABSDP1Rpqpt/n1UY54KRNJUgbaqzLMf0unEae6gCgpD1HqPAxB5WBEo5CQLprjkIjFYZO5d+OIaIkzIrJRDRD19EMOqLx1Ute2narsgbB8WP215zxO5xbj0asSK47X7Ldj22vbevnPA/bBouLyX1TMtll1+AKuevMfGE0CmfnFSAQw8Rvzh/jujDycuFDoto1rYxzE8oNZdo//OVx522f6T3sw+9YuuOnD9eY6vjDCLiS4c+07awEAWbpSvHtnD+vxad/vQqG+HC/8tt8uWAghMGfJIcSF1+9u/h+2ncGJC0U4ceFknQaLxsSyqeM3G0/hhRvYxjX+hNm6dSs++eQTdO3a1Zv1UYw2UU2vKFhcrhw+OCZa4phoiT9NydbjTVCCdtIZtFdloL1UEThUpxEmFSJROo9EnEeqeou1vElIyEQoMkQz8x9TJDJEZMXjSGQhlOM46oF/Dl/Ajd1irPuqJCeGo1NMkEerg9bEVxvS8XxqB7veFMvsms0nc5HY3L58VcGt9bOLMeP6jrh/kOOuwRZCmG9ZPfLtdnRtEYxpI9p7VGfb3XAX7TyLM5cqb0VdLNJbg4UQAr/uOosuLULQpllT7DmTh23pl8y9kTbdVK5uKV3unyMXHMZfWa5yyGaq+uXh6mROEaKD/eqk21wIAaNJnrVzamOGkTeC6mdrTiA+ogmGdzT3wBXpy9FEW72PQpNJIO1gFrq1DEHzYMfbbrwtZlajYFFYWIjx48fjs88+w6uvvurtOinC7b1aYv5m8wyCObd2wTO/VH9qoyeK4I+doi12GtvaHBWIRB6SLGFDZe7dSJTOI0DSIwa5iJFy0Q+HgMt+tumFD86ISJwR9oHjdEUQyUcTcDJd7ft99zn8vrtyPMj9X21FbnEZvploP+3g5+1nkKkrxaSrEq0BwFOXj0Np42Ka9YHzBTio9fx7/8qfB3Bnn1jr/wdn5w9nmW+n/HP4gjVYXCzU49dd53BLjxZQScCTP+7B7b1aVvmj23bsSXaBHgmRTQEAf+09jye+3w0AODn7Otz4gblXY/WRC0hqXjlOyXL9jNxitAz1d/nPfdr3uxyO5erNA2SXHajsVXlvxTGsO3YBnVsEo6nWB++vPIZ2UU1xa8+W+H3XOSx4sL/TXhWD0QQflVTjD9Invt+FfWfzcVP3GI+fW240IVNXipah9aM3ptRgxGMLdiIzvxS/Th7oMBuqurafuoRZiw8CMI/7SDuQhQe/2YbHr2lTrUD7844zmP7THvioJByrhTVylKJGwWLy5MlITU1FSkpKlcFCr9dDr6+8X6zTmZO8wWCAweC9IdSWa3nzmlWJCtIiS+f8XniX6Kb4ckJPxIUGoFV4gDVY3NwtGq/d2hntX0yrxZpJuIBQXDCFYi26AtbPDYFw6BAnZSNWuoCWFX+bH2cjRroIrVRu7elwRif8kSGaIV1EIV00N/8xmf++gGAwdNSOi0XmNRLu/mKL3fF//2j+oIwJ8sW0H2sWXqs7ZqigtBwfH6rZb9mdXlzq8pwlVFgYDAaUG03oVbH/y7L955EY2QTLD2Y53AYBgO+3ud6fZsqCnVj31FAAwA6bHsRJNmNvLNNyra9fbsJ7yw/jreXmsUxxYZUDS/MKS6y/3V4qdvxZ895+H+wrt1+/5O3l5mXubQPPkaxCzFlyCADwyepjeCKljd1zcgr1uPqttRjeoRneGl11r7CuxIBDWQXo0yrUGkQW7TxrfX/W9+bm52O50YQ1xy6iV1wIHlu4GxtP5GLehF4Y1Ca8ytd3xmTTU2L7897Tn9ErDmXjke92WR/vOZ2Lzi2CalSnc5cqV+o1GAx4/lfz/5n3Vh5DSlIkjCbh9tqW227lJuH0fRiNphp/BsnxGeap6tbN42CxcOFC7NixA1u3bq26MIDZs2fjpZdecji+bNkyBAR4Pw2npdXmB7bZza0kLD2jwoT4IuSVSVh5ToVjOvsP1MWLFwMA9lf8sTR11vmzWLIkA/KMm5VwEcG4KIKxU7R1OKuGEdFSLmIrgkasdAGxUrY1iERK+QiSStBJOoVOOOXw/ELhh1MiCidFc5yqCB4nTc1xiqGj1j1Rw1BRHy1evBgfHlDBsuPA5pOXkJ2Ti5r8+8kq0Fv/L55Ir7zmkv2OAcUi41KJNVQAwOncyl12Fy1ehgA1cKYIcPV/eOXhHKfHXTl89BgWl5nDhxDmcZ9pZyWUGtT4Y08mUpqYl4NfkiHhcL4KkzoY4XtZvnt5hxoX9RLGJxrRJUzA36eyfhcvVrbdwl8XI8hmTLdJmAcrGwUwbZO5fHN/gcwSc/m3ft8KXZIJ23Mk5OmBYS1c9xcZTYDa5i5quk17//nXYuugaE9/Rj+7VQ3b7/269etwumnVzzMKc3vaTmradVGCpZv21z8WQ19aee0bPzIPun69bzm0LvLz+XOV78ny78rM3HbHT6Zj8eIrW56gLj7Daqq4uHp7X3n06ZaRkYEpU6YgLS0Nfn7Vm9b17LPPYtq0adbHOp0OsbGxGDFiBIKCapY6nTEYDEhLS8Pw4cOh0VRvsFZNXQfgNZvpZ08KgXYv2P9juO46+26yKRuXAQBiY2ORmtoJUzctq9U61oQRauttkI3o5HDeD3q0lC6glZSFeCkL8VImWklZaC1lIkbKQVOp1GXoKBD+FWHDHDhOiSicMkXhIoJwUQQhH004toMAAJ37X4UjG+2n+J4sqHkotfxf3LXkMP457/hv0xODh1yF+7/ZgVMuphTXREJCAq4b2Q47M/Lw0P924p7kOPx5unLBNUv9p8ww/8woi+6Km3u3tJ4vKC3HxY0rAQDfHVcDx4H37+wGbDT3ZoWFheJEQR4A4O+8KNzWMwYdo4OgVgFjPt2CiQNaoU2zpsCmXQBgDRUA0CwqCoOv6Ywps8wLgD14Q7LdbaMjWQVoEeKPhdvOYM7fRzC0XQQ+Hd8DKpWErX8exNosc2/SB8eD8cejfWv0M/qFXSuB8srZUL9nBWPxmIFunyOEwDVvr0NBqQEbnrrKOmVatT8L846Y22X6Fucff/rmXREcpMU17R2ney8r3ANcNE+n7thvKOLDzTNALD/f12ep8NW/roXBaMLKQxfQOz4U4U7Wi3GmLj/Daspyx6EqHgWL7du3Izs7Gz179rQeMxqNWLNmDT744APo9Xqo1fZRT6vVQqvVOlxLo9HUSuPV1nWrcv+g1taR9JZ6OKNWqxzOvX5bVzz1s3zLP1dXKbTWAaSX84UBsVI24qVMh9DRQspBoFSCzlI6OiPd6bXLhQqX0BS5Igi5IggXEWj+GoG4WHEsF4HIFZXHjZcPECFFGPa2Z+uGVGX0p1vw7KgkzNtwZaECAH7dnenVUAEAqoqfCY8v3IO8EgPeW3nc7vyiXZkY0yfW+tgECRqNBtkFpfh6QzpWHHRcDG3aj5U/T2zHaKw/fhHrj18EAKR0iMKlYgP+u/wY3h/bw+EaFc9Gz4pQAQAFZSbrz6+Nxy9i7Geb7EqvPpKD3/ZmYUzvWEhS5S8Kxy8UWZ/nyc/o0xeLkV9iP8X6aHYR1Gofl+uKAOYBx2cumXuaXvzzEN4c3Q0A4KOu+mfGjN/N05kPvDwSAb72H5EqVeV7Gv7OepycfZ3DGBiNRoNP1h7Fm8uOoGWoP9Y9fU2Vr3n58921j9EkUFxWjkC/uv+cq+73zaNgMWzYMOzda9/let999yEpKQlPP/20Q6hoTJ4ZlYROMUGY9sNut+UGtokAANzdvxX+t8n8g25Mn9gGESzcKYMGx0ULHBctHM45Cx3xUiZaShcQLhUgSCqGj2RCJHSIlKq/8VueaGIOHQhEnghEnmiCPDRFnmhq83cTu3NF8ANvyTQuuzLycMenm6ouWA0XXawvciWW7c/Cs6M6wORiFsVTP+9BoF/lj+q3l5sXcJvx236X1zQYK691+UJqFrav9/rSQ07LuNuldfaSg87r+9Me3Nzd8eeAO0aTwPZTl9C1ZbDdbBlX2873/c8K3NazBZ51slx9ek4R/tpbOUbsp+1ncOJCIZ6/vqNHczb0BhMCbDobnM1y2XD8ovVnuq03l5lvbVnCzZlLxdhxOg+pXaJrPPDU4o5PNmLbqUuYeUNHjOjU3ONVa+uCR8EiMDAQnTt3tjvWpEkThIeHOxxvbDRqFW7t2RJdW4Yg1MkI743PXoP9Z3UY1sG8/PDtvVpagwUALHyoP1YeysY1Sc1wp5d+CNYX7kIHAGhQjlAUIFzSIUzSIRwFCHP4ugDhMB8LQRFUkkCIVIQQqcjlQFOndRFq5FeEjktoinzR1C6Q6BAAnQiADk1QIPyhQxPoRAAKEMBQQliwxfVA0Zo6mVNUZRnLCpaAeUM8d6GiulbaLPueYTOOxFZOYZnd46d/3gONWoWk5oHYc8b1tNwft2c4TL00uZm19MHKY3h7+RFc3T4Sb47uBh+VCv6+apQZnU+VzSnU45M1J5DaNRpdW4bYnRvx9hqH5+04nYdbP9oAT9h2RLz42z78tfc8ooLshwDkORnAu+qwfQ9SfrEBg14z9/o8/dMevHNnd4zsZJ63nVtUhru/2AwftQrz7jH3Gu07q8PvezMxdVg762yhLSdzsfJQNp4Y3ta6Js3MPw5g5h8HsHfmCLz650Hc2D3GaciRQ+NZKamOtGnmfFRRdLA/ooMrk+XlqbV/Qjj6J4Rj52nnv10omQE+yEYoskVotZZhVMGEEBTaBY8QqRAhKEKwVIhQFJofS4UIRhFCpQKEoBBaqRy+khGRyEekVL21CmyVCxUKEOAQOKx/IwA60QQF8Ide+EIPTeUfobF7XGb32Bcmji9p9Go6XbguWQLIiQvuw9CcJYdQcNkqse1fTMPtrSXYjj4rLitHfokBX29MB2BeF8UyG+jZUUlV1ufGD9YjfU4qcgr1WLTjLG7t2cJlGPHUC7/tx/SR7REbFoCvN5p/Cbw8aD22YAcmz7d/nu0+OADw0P+2Wb8uMRjx8P+2W5c4//cPu6x7F/WZ/Q9iA9RI32j+xTK/xIC3xnQHAIz5xDywtEjvuPLuW2lH8P22DHy/LQPpc1KRV1wGP41a1uXFrzhY/PPPP16oRuPTMtR591V4E8fxKFX57oF+GF+xaVZjYIIKuTCPuzgGVHNNaAE/lLkIHZWBJFAqRhCKECiVIAhFCJKKEYRiaCQjfCQTQlGIUKkQgOcrerpTLlR2QUMvNCiDD4xQwwgVyqGCCSoYoTIfE5avVSiH2uacfZnyiq/10KAEWpQKX5TC/KdE+JqPVTwutT7WoERUHtdDA/bU1D7LlGIluDxUWPx0Uo0Tn23BvPv6Ithfg56vpDndoA8AZi9xfnvmckaTwANfb8OujDykOZmOXFO/7z6HVYezsXPGcJdlqpMF3S1tvqFivAtgfh/phZX/z37ZcRa6knK76b62vdwW89anW7/+1/wd+HPPeQRqfbD3pZFVV66WsMdCJiEBvlj2xBD4+dinyrjwALxxe1dM/6lyzMXYvrFuu2DrS/dX/SahFFqchxbnRbiHG1SYQ0kQihEoFSO4InAEotgaPCyBJEgqRlOUQAsDtJIBWpTBF+VOH2ukyoWpfCQTfKBHE+gt1a03TEKqCBkalEGDMuGDMmhggA/0qPha+KAMPjBUPNbDp+KYxv64MH9tqtYbrE4ZURGrBFQQkCAqIpiASrIcN9mcs5Q3WR+rYIIBPtZwZQlTzh7rhabysU1I00OD8iv4cWq7T4zS7Tidhye+34VZt3R2GSo8kfh/ldM+t3h5f5KC0nKXC8Z5Q1Vrn7lau8WVP/eYbwsX6MtxoUCPyEDPf1H1BgYLGbWLcr4r6ejesTCaBJ75ZS+eT+2ABwYn4NrO0dh3Nh9vLD3s9Dm39GhhXRAHAB4f1hbv2ezSSVfCHEpKoTXfrgFqvr+4DTWM8IXBJniUmf9GObQog1YyQAUTfCo+CH1gtD62fDj6SEaHMtZzFX9rJPP1/VAGP5TBX9LDDwb4Qw8/qeIY9BXnyqBFGfxRZg0+KkkgAHoE1MPQU5+UV/QiAVJFbJEgAIiKx6h4bI45qDhf8SdTgtBWnjNBBZOQKr7jlT1Rpsv+tnztqqxtgBMVX9v+0xV230zX58uhRllFKDSHxIqv7QJl5TFL6DTY/CkTPpAgoJGMKDuyF8++thTDVUb4wAgNjOZoJlX8jcrjPtbjlmOVvSG27eXQpnbHK/4Wld8by/fEZP2jsh6vjJ6S02MmYf/Y0nK2AVYlmUOT3TGYj+34cR9+23UWY2GCSl1Zw8qAXBmW1ZLJ7nFlWK4MzOrLHqsgAH1fINBxymxdYLCop+7sG4frukYjqGJK0dB2kRjaLhIlZUZ8sOqYQ/m3xnRDp5ggvPrXQTw+rC2mDW/nUbCY/2A/jPus8dxOqQ+MUKMEaliHzV0eVmS+5e6D8oowYoCfpId/RW+LJQxppMqvfVEOX8kAjeVrmL/WSpVfW49L5VVmE6kab16CsP4oNQqV9QNCQGX3YWGC4zmj7QeHkKCRyq3By6+iZ8kPFX9XBL7K8zaPpcrBe+ZeJy/uy8EAp1z7gZ61PFs0p9z5gNy6wGBRjwU5maf87xHtcHOPGPx32REs2ZdpPS5JEh4YnIBberRAeFPPur8sA4mWPTEEI95eU63nhARonI6IJuUohw8K4YNCwHnIqf9jDWudBFNFj5M5iKhtf8+VLL/vwtp3YPv7tO1jld0xk90tGzVMUEuVX1uP2ZSzPa6SKs9fHtBsH9t/7fq8ea1Kc0+Br1QODcqhselt01SETHN4rChXESAvPyYgoRzqil4MtfXrcmH5Wo1y+FR+LWzLVj7H3Etg23aWvgdc1p7CbRtbvzdS5TFLj0BlL4Pt8x3LqGGy9mrY9nA4HrP0rjg7VtmzYgnJznpKbHui7M+rKnpRKt/ZfRr5tm9nsGhgJElCm2aBePGGTrhQoMc9A+LtztuGiv4JYUjPKUamrrRa124XFYj/3d/XYT8KZ0Z1bl4rU++IGhIBlXmwLXzhsAKLN4MXQxx5aKK2Guue1xLOcWugmgf74adHB+DGbq53LlzwYH+se/pqp+feuaM7vpjQG8ueGGJ3fHDbSBybNcqh/OPXtMGhV661PradOguY1+mYdFUifn402e74okkDMKSdPPf5iIgaKznvpDFYKJgkSfBRqzD/wX5OzgHDOkQ5HUDqo1Zh07PD7I7d2L0F/DRqzLuvD8b2jcWDgxPw52OD8PS1Sfht8kBEB/vjqWuT0KtVmPU5c8f3RI+4UMy7tw+6tAiu8fuIDq7evjRERCQ/BotGYEBiBNZNH4InuzifW+5M88s+zJtWbBV9dftmmH1rV/j7qtG5RTAevSoR3WJD7Mo+elUi+rUOQ0rHKADmxcAimlZvIx4AGNIuEgse7A8AuKN3LDY+OwzLnhiCa5KaVfsaRESN2eV7mNQljrFoJKKC/BB7BbfcLg8a7jx9rfsV85KaB+JQZgEAIDJQiwsF9vsvSACSE8Nx4OWR8K9YPa5dVCDevqM7ur1k3kWwU0yQdcU6IiKyx1shVK/Fhwdc8TUeHpoIAEjtEo0lUwZbjycnhLt6CgJ8fexSd7C/Bvckt8JtFXuyEBFR/cMeC6pSbNiVB4v+CeHYMWM4QgM0kCQJy54YgrQDWZg4sDV+333Orqy7HryXbzJvdvd/i/a6LlQH1CrJa3s7aNSS3W6URERXSsY7IeyxaKyqsyfJT48k4/qu0Xjj9m5eec2wJr7WHoh2UYGYfHUb+Puq8e39/dAuSr6pUc58M7Gvy3OjOjfHfi+uwz81pZ3XrkVEBMC6HqgcGCwamXfGdMW/rm6DgW1c34Kw6B0fhg/G9fRofEVNDGobgWVPDLXO/riuc3SVz/Hkv8zGZ6/B9JHtAQA94kKq9ZzBbSPw3tgeWDRpAD4c1xPdbQaozr2rl8POgb9MGuBBjaq26wXXGx8REdVnDBaNTGqX5nhyZHtZRwy7smTKYHz3QD/c3qtllWX/dU0bhDXxxaNXJSKpufM9Vyyig/0x+eo2ODprFB4ekmg9/tg1bVw+R5Ik3NgtBj3iQpHaNRodol2/RlOtD3rGhToc7x4bghX/Hoqbu8dg5g0dXT4/IaIJbutp/55DAqo/i4aIyIGMP+I5xoLqjZAA32rv1Bod7I9tz6VApZKw6lC29fhrt3VB39bhuPrNfwAAX9vc0tCoVXb3Hf89oj02HL+I7acuoXerEJzIvIRcvfP/jcLJEIgbusXgj93n8MjQBADA5/f0Rm5RGfq2DsOp3GIMrVgY7J07ewAAZv5xAABwc/cYdIoJRpeWwdh/TodrOzdHbnEZft5xplrvHQB+fjQZt83daH385Ih2KNQb8fHq49W+BhFRbWCPBTVYKpU5BLw5uhsC/Xww84aOuKNPHFpHNEH6nFSkz0m1fri78undvfB8agd8MLY7ro42byAV5OeYt2/q3gIAkBBZuf7+W2O64c/HBmHSVeaej5SOURjTJxbxEU3cvm775kF4cEgC+ieE4/5BrSFJEu7sE4cO0UEun/PxXT3tHvdqFYZFNrdferUKwzOj3E/zdeWN27t6VH7ltEHo1tK84Nngtq6D4OW9MM7c3b8Vvr2/H/bOHOFRHYjIPTk7pdljQQ1e5xbB2P3CCGvQcOfyEuFNtXhgcAIMBgMGNRcYNbgHesY7flgmJ4Zj5b+HIiakcilzjVqFzlewoqgttUrCokkDcN+8rbiqvWMo6dnK8VZLD5vbL5YfIl1bBmPPmXynr9G5RRD2nXVc+2NEx+aYjj3VrmtsaADm3dcXS/dn4vqu0fhm4ym8sfRwtZ9v0btVKF65ubPHz3MnOSEcG09c9Oo1iRoiX7V8/QbssSBFqE6oACo/jH2clFdJ5u3pw5o4H9+QENnUYdBmTbRv7nwGjJ9GjQUP9beu+WERElD1/sqWd/PFhD7oHhvitCfB1ShxrcbzHwNhTXwxtm8cAv00uKVHC6dlRnVu7vYaPzxsv6/M48PaelyPyzlbvt7W3PE93Z6/3ITkVtavZ97Q0WFvHaL6isGCqI5EBmqx5f+GYdeLdd/1/udjg/DabV1wdfvqLU3eN96878roXi3tdrfc/H/DHMomNjOHlchALX6dPBD/u9/5/jA/PpKMG7rFYNOzwzBteDt8NL4n/DRqPDCotbXcP09ehXZRTdG1pWNvjFblONhE5aLPtU98GDrFuL69c3kYfHRoIjq3cF2+OtwNSk6fk4pRXaIxslNUta93Q7cYPJHSDmP7xmHCgHi0iwq0Lm8/ILFyZtVDQxLQv7Vjr9LKfw+1fj3vvj7Vfl1vee66DtUqV9UAaGpYNGqp2r9s1QbeCqFGp1mQPJuadW4R7NGtky/u7Y0tJ3MxuG0kSgxG63HLBxsAbHluGIr0RkQ0dVyX5IeHk/HpmhNYfjALgLlXo098GPpUBBbbHoLHrmmLz9edRFLzQMRHNMGyJ4ai3GjCzzvOoG/rcBzO1OG9FUdxc9Qlh9fxs+nxmP9AP4z7fDMsL/jr5IFYtj8LfeJDUWIwwt9XjYe+2e503xd/XzX+fGwwdmfkQQBYfiALH6w65lBuakpbvLP8qPvGc+PDcT2x+0wejmYV4kKBHv9NO+JQZsGD/aGSzFOue8eH2Z3b+lwKisrKEdFUi/hn/gIADGkbiSdTEjHoP8uQXVr5Az0hsinWTL8aBpMJiZGer9UysE04YoL98eP26g/stZia0hYPDklAh+gg3PXFZrdl59zWFTd/uN7j1yDn5o7viY0nLuKbjadkef09L3pvnZ2aYLAgqqcC/TQY1sH827Wvjwqv3twZapWEJjbBolmgH+Dil82+rcPQt3UYxnyyEVtO5mJcvziXrxUcoMGBl0dC61N5q8dHrcIdfczPaR3RBMPaR2Dx4sUOzw0J8MUL13eEWiUhwebDU5LM41BSu9qvS/Lr5IFu37dlU7vusSGIj2iCJ3/cbT13TVIzTLqqjUfB4vZeLe2mFvuoVejVKsy6E+8dfWJRXGbEVRUziQCgX+swl7/x+fuq4e9rbqfN/zcMJy4UITkxHAaDAY91MmLGdvsfq3EulsQf07slftjmPjDMvKETisqM1mBx38B4/LH7PHIKzfvrTB/ZvsrxLYNcDLBt26wpjmYXAjDfBnSnS4tg/P6vgSg1mLDj9CWM/9w+qLw5uhsWbDkNo0ngh4eTYTQJdHjhb/cX9cDzqR3w6l8HvXa9y315b29M/Gqbw/GHhiTg0zUnrI8fu6YN3l/pGHYvN6pLNEZ1ifZKsHhoSAI0agkfrqr+jC/Lv0+58FYIUQNxV/9WGNvXdThw5ZuJffHnY4Mwpnes23IBvj5Q17D7dOKg1pgwIB4C3l2a/JYeLTDpqkR8M7Ev0uek4st7+8DXR4Uv7+2N/47uhkOvXOv2+V9M6I03bu+KVuFNXJZpFuSH+Igm+PiuXujXOgzrn7mm2t3IUUF+SLa5JRLkCyx4oHq3PEZ1icbx/1xXZbluLYPx8JAEvHpzZ7x4QyfY3hebfLXrtVhc3aKy+OTuXtavo4P9kRBhbqOtz6U4lNX6qCBJEvx91RjYJgLv3tkdIztFwVetwv2DWuP2Xi3x86MD8OvkgfD1UTl8sHVzclvNEw8MTnB6a85bJElyOj7n/y67lWS7vkxHN7O4rkSXy24HThzYGtNHJjmMSarP2GNBpHB+GrXXZq9UxXa9D2/c4VWrJDzlZLfca5Jcj5NoEeKPs3klAGDt8amOazs3x7VVDDitjt6tQvHf0d3QOtIxzPzv/r64+4stAIDEiKZQqyTcOyAeX21Id3otAfOH3rM2H3DThrfH/y3aizv7OAbF/7suCf9ZfAiA6/b/7+huyC0qQ0JkUyydOgRFZeWIDNRi5ZNXWcvsnDEculIDtqZfwoerjmHObfZTkm/q3gI3dW+BUoMRWp+qfz/1JG6mdIiy3r6z9fmE3vhp+xmM7hWLPrOWe3DFqkkABiRGYN3TV2PfWR0e+Xa7y3IWAkBsmD8yckvsyqx/5porqsvlgdCScfu2tr8l99V9fXDvvK1X9Fq1hcGCiGqFXKu7fvtAP8z66yAmXZ1YdeFacpuL1WMHt43E8mlDkVtUZr1FMuP6jlh79AKOXyhyKO9sYbZx/eIwuG0EWlRMfV4+bSi2pufi+q7RCPTTWINFderW3sWgzdAmvght4otW4U3croRb3VlSs2/tgjs/3YSU5nosSnf/nJt7xCD9YhGOVdymsWgW6GddM+aj8T0x6bsddudTu0RjwoB4NNX6oGNMkHX8CwAsfKg/7vx0k8vXtPxbbRkagOhgf6R0aIa2Ue4HtI7rF4fP11beJukWG4JZN3e2fl9qyva/zR29Y52OCXv0qkRcZTMIPKVDFLam5yK/xHBFr+0tDBZE5DXNg/zQuUUQNGoVmsh0n7d1RBN8PqG3LK9dHW2a2Q/iVKskrPj3VXjm5z1YuDUDQX4+0JWWA4DLW0u2Ow63adbU7pptmjXFsexCjOpSObbly3t7Y9oPu/Hf0d7ZULA6LINsX7yhIzrFBGPbs1fj77+XYFG66+cMbBOO1C7R6BsfhsV7z2Nwu0inH9TXdYm2W5dl2vB2mJAcj2AXU7P7J4Tjj38Nwo7Tl3A2rwSfrjmBplofFOrN7WwbgdUqCZ9PqLyd1a1lMHafyXeYOXNXvzh8YRMsfqti7NCtPVvgTG4JtqTnui1nW5fXXCxe1yfePAPp07t74UKhHuP7tULXmUvdXrcuMVgQkdeoVBJ+nzwIklR3PRaRgVpcKNBDo65/+994YuaNndC5RTCuTmqGgXNWAjAHNU8tfnww8krKzAN7K1yTFIWdM4bXaS/S1JR2GNcvzlqP6oxbGdmpOSRJQrMgP9w7sLXbsnPH98Kcvw/hocEJ1gG/7nRpaV5Gv9RgRGxYAK5uH4lBr60y181Nu3x2T298t/k0xvaNgyQBL/95AL1ahUKSJPRPCEf6xWIEuAjRAb5qFJeZZ3S9NaY7jCaBJfvOY+bvB9Auqik2HL+yxdxGdKq8dZfYrCl2ns67out5C4MFEXlVXc+f/+6Bfnj970MNfvt5P40ad/U3L8i14t9DUVJmrNFmdL4+KrtQYSHHrSln9Vjz5BDM33oWEwa0QvJsc4DqEReCbi1DPBqcHBsWgA/HuV7w7LnrOmDW4oN4+rIxOn4aNe7u38ruWLMgx+nalef88MTwyn9b+18aCf+KW0DPX98RrSOaYJSLHZmfvjYJL/6+3/pYrZJwfdcYXN81BgCQX2zA8LdXI6dQD1NF51SZ0eSyLu68P7YH/rvsCO4f5D6Q1QUGCyJq0NpFBdp1XStBTda8aCiig/2s+9p8/1B/rDiUjWnD23llVVtbDw5JwC09Wzhd48Xiy3t7IzNfj3ZVjKewZTvdu6nWx2GlXFt39InF2qM5GOpkmX7APM1707PDoFJJ1jEh4S5W/rWlcbKqZsvQALx9R/cqn1sXGCyIiEgW/RLC0S8hvOqCNeQuVADuZxd5g59GXeV4n8t7+KZc0watwps63el58tWJOJxZgAGJ1dsFWi4MFkRERDL74cG++OufjejaMhi9WjsPDtNH1mwH47rGYEFERCSzHnEhOB/h3QXm5MKVN4mIiMhrGCyIiIjIaxgsiIiIyGsYLIiIiMhrGCyIiIjIaxgsiIiIyGsYLIiIiMhrGCyIiIjIaxgsiIiIyGsYLIiIiMhrGCyIiIjIaxgsiIiIyGsYLIiIiMhr6nx3UyHMu7fpdDqvXtdgMKC4uBg6nQ4ajcar11YCto97bJ+qsY3cY/u4x/ZxryG0j+Vz2/I57kqdB4uCggIAQGxsbF2/NBEREV2hgoICBAcHuzwviaqih5eZTCacO3cOgYGBkCTJa9fV6XSIjY1FRkYGgoKCvHZdpWD7uMf2qRrbyD22j3tsH/caQvsIIVBQUICYmBioVK5HUtR5j4VKpULLli1r7fpBQUH19ptSH7B93GP7VI1t5B7bxz22j3v1vX3c9VRYcPAmEREReQ2DBREREXmNYoKFVqvFiy++CK1WK3dV6iW2j3tsn6qxjdxj+7jH9nFPSe1T54M3iYiISLkU02NBRERE8mOwICIiIq9hsCAiIiKvYbAgIiIir1FMsPjwww8RHx8PPz8/9OvXD1u2bJG7Sl43e/Zs9OnTB4GBgWjWrBluvvlmHD582K5MaWkpJk+ejPDwcDRt2hS33XYbsrKy7MqcPn0aqampCAgIQLNmzTB9+nSUl5fblfnnn3/Qs2dPaLVatGnTBl999VVtvz2vmzNnDiRJwtSpU63HGnv7nD17FnfddRfCw8Ph7++PLl26YNu2bdbzQgi88MILiI6Ohr+/P1JSUnD06FG7a+Tm5mL8+PEICgpCSEgI7r//fhQWFtqV2bNnDwYPHgw/Pz/Exsbi9ddfr5P3dyWMRiNmzJiB1q1bw9/fH4mJiXjllVfs9kVoTO2zZs0a3HDDDYiJiYEkSfj111/tztdlW/z4449ISkqCn58funTpgsWLF3v9/daEuzYyGAx4+umn0aVLFzRp0gQxMTG45557cO7cObtrKLKNhAIsXLhQ+Pr6ii+//FLs379fPPjggyIkJERkZWXJXTWvGjlypJg3b57Yt2+f2LVrl7juuutEXFycKCwstJZ55JFHRGxsrFixYoXYtm2b6N+/vxgwYID1fHl5uejcubNISUkRO3fuFIsXLxYRERHi2WeftZY5ceKECAgIENOmTRMHDhwQ77//vlCr1eLvv/+u0/d7JbZs2SLi4+NF165dxZQpU6zHG3P75ObmilatWol7771XbN68WZw4cUIsXbpUHDt2zFpmzpw5Ijg4WPz6669i9+7d4sYbbxStW7cWJSUl1jLXXnut6Natm9i0aZNYu3ataNOmjRg7dqz1fH5+voiKihLjx48X+/btEwsWLBD+/v7ik08+qdP366lZs2aJ8PBw8eeff4qTJ0+KH3/8UTRt2lS8++671jKNqX0WL14snnvuOfHLL78IAGLRokV25+uqLdavXy/UarV4/fXXxYEDB8Tzzz8vNBqN2Lt3b623QVXctVFeXp5ISUkR33//vTh06JDYuHGj6Nu3r+jVq5fdNZTYRooIFn379hWTJ0+2PjYajSImJkbMnj1bxlrVvuzsbAFArF69Wghh/oes0WjEjz/+aC1z8OBBAUBs3LhRCGH+j6BSqURmZqa1zNy5c0VQUJDQ6/VCCCGeeuop0alTJ7vXuuOOO8TIkSNr+y15RUFBgWjbtq1IS0sTQ4cOtQaLxt4+Tz/9tBg0aJDL8yaTSTRv3ly88cYb1mN5eXlCq9WKBQsWCCGEOHDggAAgtm7dai2zZMkSIUmSOHv2rBBCiI8++kiEhoZa28vy2u3bt/f2W/Kq1NRUMXHiRLtjt956qxg/frwQonG3z+UfmnXZFmPGjBGpqal29enXr594+OGHvfoer5Sz8HW5LVu2CADi1KlTQgjltlGDvxVSVlaG7du3IyUlxXpMpVIhJSUFGzdulLFmtS8/Px8AEBYWBgDYvn07DAaDXVskJSUhLi7O2hYbN25Ely5dEBUVZS0zcuRI6HQ67N+/31rG9hqWMg2lPSdPnozU1FSH99DY2+f3339H7969MXr0aDRr1gw9evTAZ599Zj1/8uRJZGZm2r234OBg9OvXz659QkJC0Lt3b2uZlJQUqFQqbN682VpmyJAh8PX1tZYZOXIkDh8+jEuXLtX226yxAQMGYMWKFThy5AgAYPfu3Vi3bh1GjRoFgO1jqy7boqH+f3MmPz8fkiQhJCQEgHLbqMEHi5ycHBiNRrsPAgCIiopCZmamTLWqfSaTCVOnTsXAgQPRuXNnAEBmZiZ8fX2t/2gtbNsiMzPTaVtZzrkro9PpUFJSUhtvx2sWLlyIHTt2YPbs2Q7nGnv7nDhxAnPnzkXbtm2xdOlSPProo3j88cfx9ddfA6h8f+7+L2VmZqJZs2Z25318fBAWFuZRG9ZHzzzzDO68804kJSVBo9GgR48emDp1KsaPHw+A7WOrLtvCVZmG0lYWpaWlePrppzF27FjrJmNKbaM6392UvGPy5MnYt28f1q1bJ3dV6o2MjAxMmTIFaWlp8PPzk7s69Y7JZELv3r3xn//8BwDQo0cP7Nu3Dx9//DEmTJggc+3k98MPP+C7777D/Pnz0alTJ+zatQtTp05FTEwM24euiMFgwJgxYyCEwNy5c+WuTq1r8D0WERERUKvVDiP7s7Ky0Lx5c5lqVbv+9a9/4c8//8SqVavstqBv3rw5ysrKkJeXZ1feti2aN2/utK0s59yVCQoKgr+/v7ffjtds374d2dnZ6NmzJ3x8fODj44PVq1fjvffeg4+PD6Kiohp1+0RHR6Njx452xzp06IDTp08DqHx/7v4vNW/eHNnZ2Xbny8vLkZub61Eb1kfTp0+39lp06dIFd999N5544glr71djbx9bddkWrso0lLayhIpTp04hLS3Nbkt0pbZRgw8Wvr6+6NWrF1asWGE9ZjKZsGLFCiQnJ8tYM+8TQuBf//oXFi1ahJUrV6J169Z253v16gWNRmPXFocPH8bp06etbZGcnIy9e/fa/WO2/GO3fOgkJyfbXcNSpr6357Bhw7B3717s2rXL+qd3794YP3689evG3D4DBw50mJ585MgRtGrVCgDQunVrNG/e3O696XQ6bN682a598vLysH37dmuZlStXwmQyoV+/ftYya9asgcFgsJZJS0tD+/btERoaWmvv70oVFxdDpbL/kahWq2EymQCwfWzVZVs01P9vQGWoOHr0KJYvX47w8HC784ptI1mGjHrZwoULhVarFV999ZU4cOCAeOihh0RISIjdyH4lePTRR0VwcLD4559/xPnz561/iouLrWUeeeQRERcXJ1auXCm2bdsmkpOTRXJysvW8ZTrliBEjxK5du8Tff/8tIiMjnU6nnD59ujh48KD48MMPG8R0SmdsZ4UI0bjbZ8uWLcLHx0fMmjVLHD16VHz33XciICBAfPvtt9Yyc+bMESEhIeK3334Te/bsETfddJPTKYQ9evQQmzdvFuvWrRNt27a1mx6Xl5cnoqKixN133y327dsnFi5cKAICAurddMrLTZgwQbRo0cI63fSXX34RERER4qmnnrKWaUztU1BQIHbu3Cl27twpAIi33npL7Ny50zqjoa7aYv369cLHx0e8+eab4uDBg+LFF1+sN9NN3bVRWVmZuPHGG0XLli3Frl277H5m287wUGIbKSJYCCHE+++/L+Li4oSvr6/o27ev2LRpk9xV8joATv/MmzfPWqakpERMmjRJhIaGioCAAHHLLbeI8+fP210nPT1djBo1Svj7+4uIiAjx73//WxgMBrsyq1atEt27dxe+vr4iISHB7jUaksuDRWNvnz/++EN07txZaLVakZSUJD799FO78yaTScyYMUNERUUJrVYrhg0bJg4fPmxX5uLFi2Ls2LGiadOmIigoSNx3332ioKDArszu3bvFoEGDhFarFS1atBBz5syp9fd2pXQ6nZgyZYqIi4sTfn5+IiEhQTz33HN2HwKNqX1WrVrl9OfNhAkThBB12xY//PCDaNeunfD19RWdOnUSf/31V629b0+4a6OTJ0+6/Jm9atUq6zWU2EbcNp2IiIi8psGPsSAiIqL6g8GCiIiIvIbBgoiIiLyGwYKIiIi8hsGCiIiIvIbBgoiIiLyGwYKIiIi8hsGCiIiIvIbBgoiIiLyGwYKIiIi8hsGCiIiIvIbBgoiIiLzm/wEZWcKdd0Bh7gAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 画图\n",
    "plt.plot([i[\"step\"] for i in record_dict[\"train\"]], [i[\"loss\"] for i in record_dict[\"train\"]], label=\"train\")\n",
    "plt.plot([i[\"step\"] for i in record_dict[\"val\"]], [i[\"loss\"] for i in record_dict[\"val\"]], label=\"val\")\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "caf2b00ee2704d39",
   "metadata": {},
   "source": [
    "# 推理\n",
    "\n",
    "- 接下来进行翻译推理，并作出注意力的热度图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "913a1278f74da64b",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:01:09.230235Z",
     "iopub.status.busy": "2025-01-27T15:01:09.229995Z",
     "iopub.status.idle": "2025-01-27T15:01:09.933883Z",
     "shell.execute_reply": "2025-01-27T15:01:09.933310Z",
     "shell.execute_reply.started": "2025-01-27T15:01:09.230209Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "model = Sequence2Sequence(src_vocab_size=len(src_word2idx), trg_vocab_size=len(trg_word2idx),encoder_num_layers=4,decoder_num_layers=4)\n",
    "model.load_state_dict(torch.load(\"checkpoints/seq2seq_layer_4.ckpt\",weights_only=True, map_location=device))\n",
    "\n",
    "\n",
    "class Translator:\n",
    "    def __init__(self, model, src_tokenizer, trg_tokenizer):\n",
    "        self.model = model\n",
    "        self.model.eval()  # 切换到验证模式\n",
    "        self.src_tokenizer = src_tokenizer\n",
    "        self.trg_tokenizer = trg_tokenizer\n",
    "\n",
    "    def draw_attention_map(self, scores, src_words_list, trg_words_list):\n",
    "        # 绘制注意力热力图\n",
    "        # scores.shape = [source sequence length, target sequence length]\n",
    "\n",
    "        # 注意力矩阵,显示注意力分数值\n",
    "        plt.matshow(scores.T, cmap='viridis')\n",
    "        # 获取当前的轴\n",
    "        ax = plt.gca()\n",
    "\n",
    "        # 设置热图中每个单元格的分数的文本\n",
    "        for i in range(scores.shape[0]):  # 输入\n",
    "            for j in range(scores.shape[1]):  # 输出\n",
    "                # 格式化数字显示\n",
    "                ax.text(j, i, f\"{scores[i, j]:.2f}\", va='center', ha='center', color='k')\n",
    "\n",
    "        plt.xticks(range(scores.shape[0]), src_words_list)\n",
    "        plt.yticks(range(scores.shape[1]), trg_words_list)\n",
    "        plt.show()\n",
    "\n",
    "    def __call__(self, sentence):\n",
    "        # 预处理句子，标点符号处理等\n",
    "        sentence = preprocess_sentence(sentence)\n",
    "        encoder_inputs, attn_mask = self.src_tokenizer.encode(\n",
    "            [sentence.split()],\n",
    "            padding_first=True,\n",
    "            add_bos=True,\n",
    "            add_eos=True,\n",
    "            return_mask=True,\n",
    "        )  # 对输入进行编码，并返回encoder_inputs和encoder_padding_mask\n",
    "        # 转换成tensor\n",
    "        encoder_inputs = torch.Tensor(encoder_inputs).to(dtype=torch.int64)\n",
    "        # 由输入得到预测结果，\n",
    "        # 注意，这里输入的样本只有一个，符合Seq2Seq模型的生成方法要求batch_size=1\n",
    "        preds, scores = self.model.infer(encoder_inputs=encoder_inputs, attn_mask=attn_mask)\n",
    "\n",
    "        # 通过tokenizer转换成文字\n",
    "        # trg_tokenizer.decode方法要求输入的是字符串列表，所以需要[preds]\n",
    "        # 方法返回字符串列表，所以需要[0]\n",
    "        trg_sentence = self.trg_tokenizer.decode([preds], split=True, remove_eos=False)[0]\n",
    "\n",
    "        # 对输入编码id进行解码，转换成文字,为了画图\n",
    "        src_decoder = self.src_tokenizer.decode(encoder_inputs.tolist(), split=True, remove_bos=False, remove_eos=False)[0]\n",
    "\n",
    "        self.draw_attention_map(\n",
    "            # [1, sequence length, sequence length] \n",
    "            # -> [sequence length, sequence length]\n",
    "            scores.squeeze(0).numpy(),\n",
    "            src_decoder,  # 注意力图的源句子\n",
    "            trg_sentence,  # 注意力图的目标句子\n",
    "        )\n",
    "        return \" \".join(trg_sentence[:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "3aa03683664d5b62",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:53:27.389488Z",
     "start_time": "2025-01-27T13:53:27.388488Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:01:09.934684Z",
     "iopub.status.busy": "2025-01-27T15:01:09.934491Z",
     "iopub.status.idle": "2025-01-27T15:01:10.139454Z",
     "shell.execute_reply": "2025-01-27T15:01:10.138912Z",
     "shell.execute_reply.started": "2025-01-27T15:01:09.934661Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcwAAAG/CAYAAADVbefpAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAWCZJREFUeJzt3XtcVXWi/vHPArkKbAQEvKBopGYqGIih46WRcmqa3zRNnerM5GUam9OoR7KZSa1Ru4mVjjpadpmOOeeM5TRpNmY3MSnLS6FYipesUFJB8AIIym1/f3+QWxHQpaCw5Xm/XvslLL77u569hf2stfZaYBljDCIiInJOHk0dQERExB2oMEVERGxQYYqIiNigwhQREbFBhSkiImKDClNERMQGFaaIiIgNKkwREREbVJgiIiI2tNjCHDp0KJZlYVkWmZmZTZIhOzvblSEuLu6C7jt06FBSUlIuSS53Eh0dzdy5cy/b+owx3H///YSEhJzze8eyLN56663Llutymz59+gV/z7qr5vBa0RwySAsuTIAxY8Zw8OBBevXqVaO8LMvC29ubmJgYnnzySc7+7YHbt2/nP/7jP2jbti0+Pj5069aNqVOnUlpaWmPc1q1b+X//7/8RHh6Or68v0dHR3HXXXRw6dAiAqKgoDh48yEMPPXTZHrM0zHvvvcerr77KypUrXd87dTl48CA333zzZU53+fzhD38gLS2tqWNcNud6rTjztmHDBtd9Tpw4wbRp0+jWrRs+Pj6EhYVx5513sn379hpzl5aWMnnyZK666ip8fX1p27YtQ4YMYcWKFa4xy5YtY9OmTZft8UrdWjV1gKbk7+9PZGRkjWWrV6/m2muvpaysjHXr1vHb3/6Wdu3acd999wGwYcMGkpOTSU5O5p133iEiIoJNmzbx0EMPkZaWxkcffYS3tzf5+fkMGzaMW2+9lffff5/g4GCys7N5++23KSkpAcDT05PIyEgCAgIu+2OXi/PNN9/Qrl07BgwYUOfXy8vL8fb2rvV9daUJCAhoUd+353qtOFNoaCgAZWVlJCcns2/fPmbPnk3//v3Jy8sjNTWV/v37s3r1aq6//noA/uu//ouNGzcyf/58evbsyeHDh/nss884fPiwa96QkBCKioou8aOU8zIt1JAhQ8yECRNcn3/33XcGMFu2bKkxbtiwYeb3v/+9McYYp9NpevbsaRISEkxVVVWNcZmZmcayLDNz5kxjjDHLly83rVq1MhUVFefNMm3aNBMbG3vB+cePH2/++Mc/mjZt2piIiAgzbdo019dnz55tevXqZfz9/U3Hjh3NAw88YIqLi2vMsW7dOjNkyBDj5+dngoODzU033WSOHDlijDGmqqrKzJgxw0RHRxtfX1/Tp08f88Ybb5w307hx48yECRNMcHCwCQ8PNy+99JI5fvy4GTVqlAkICDBXXXWVWbVqlTHGmEWLFhmHw1FjjuXLl5uzvy3ffvttk5CQYHx8fExoaKi57bbbXF/r3Lmzeeqpp8zo0aNNQECAiYqKMi+++GKN+3/55ZfmhhtuML6+viYkJMSMGTOm1nNhx8iRIw3gunXu3NkMGTLEjB071kyYMMGEhoaaoUOHGmOMAczy5csbPYMd7777rhk4cKBxOBwmJCTE/PSnPzV79uxxfX3jxo0mLi7O+Pj4mPj4eLNs2bIa3/t2/l8u5nvWXdl9rTjTzJkzjWVZJjMzs8byqqoqk5CQYHr27GmcTqcxxhiHw2FeffXV8+aws165tFr0Idnz+eKLL8jIyKB///4AZGZmkpWVxcSJE/HwqPnUxcbGkpyczGuvvQZAZGQklZWVLF++vNYh3cayePFiWrduzcaNG3nmmWd4/PHH+fDDDwHw8PDgr3/9K9u3b2fx4sWsWbOGP/3pT677ZmZmMmzYMHr27Mn69etZt24dP/vZz6iqqgIgNTWVv//977zwwgts376dBx98kF//+tekp6efN1NYWBibNm1i/PjxPPDAA9x5550MGDCAzZs3c9NNN3HvvffWOnxdn3feeYdf/OIX3HLLLWzZsoW0tDQSExNrjJk9ezYJCQls2bKF3//+9zzwwAPs2rULgJKSEoYPH06bNm34/PPPeeONN1i9ejXjxo2z/TyfMm/ePB5//HE6duzIwYMH+fzzz12P2dvbm08//ZQXXnih1v0aM4MdJSUlTJw4kS+++IK0tDQ8PDz4xS9+gdPp5Pjx49x666307NmTjIwMpk+fzh/+8IdLkqMlW7JkCTfeeCOxsbE1lnt4ePDggw+SlZXF1q1bgerXilWrVlFcXNwUUeVCNHVjN5X6thr9/PxM69atjZeXlwHM/fff7xrz+uuvn3ML77//+7+Nn5+f6/MpU6aYVq1amZCQEPOTn/zEPPPMMyY3N7fW/S52D/NHP/pRjWX9+vUzDz/8cJ3j33jjDRMaGur6/J577jEDBw6sc+zJkyeNv7+/+eyzz2osv++++8w999xjO1NlZaVp3bq1uffee13LDh48aACzfv16W3sySUlJ5le/+lW96+zcubP59a9/7frc6XSa8PBws3DhQmOMMS+99JJp06aNOX78uGvMO++8Yzw8POr8vzifOXPmmM6dO9d4zH379q01jjP2MBs7w4XKz883gPnqq6/Miy++aEJDQ82JEydcX1+4cKH2MM/hfK8VZ95O8fX1rXGfM23evNkAZunSpcYYY9LT003Hjh2Nl5eXSUhIMCkpKWbdunW17qc9zKanPcyzLF26lMzMTLZu3co///lPVqxYwaRJk2qMMTb3GJ966ilyc3N54YUXuPbaa3nhhRfo0aMHX331VaNk7dOnT43P27Vr5zqhaPXq1QwbNowOHToQGBjIvffey+HDh117dqf2MOuyZ88eSktLufHGG13vVQUEBPD3v/+db775xnYmT09PQkND6d27t2tZREQEgCvn+ZwrZ13rtCyLyMhI1/w7duwgNjaW1q1bu8YMHDgQp9Pp2gttqPj4+HN+/XJkONPXX3/NPffcQ9euXQkKCiI6OhqAffv2sWPHDvr06YOvr69rfFJSUqNnaAlOvVaceTuT3deJwYMH8+2335KWlsYdd9zB9u3bGTRoEE888cQlSC0NocI8S1RUFDExMVxzzTXceeedpKSkMHv2bE6ePEm3bt2A6hfAuuzYscM15pTQ0FDuvPNOZs2axY4dO2jfvj2zZs1qlKxeXl41PrcsC6fTSXZ2Nrfeeit9+vThzTffJCMjg+eeew6oPikFwM/Pr955jx8/DlQfDj3zxSArK4t//etfF5zpzGWWZQHgdDrx8PCo9aJSUVFR4/Nz5TzXOp1O53nv11jOLMLm4Gc/+xlHjhzh5ZdfZuPGjWzcuBE4/X9/Pnb+X+T0a8WZt1O6det2zteJU2NO8fLyYtCgQTz88MN88MEHPP744zzxxBO2/8/k8lBhnoenpyeVlZWUl5cTFxdHjx49mDNnTq0X5K1bt7J69Wruueeeeufy9vbmqquucp0le6lkZGTgdDqZPXs2119/Pd26dePAgQM1xvTp06feywJ69uyJj48P+/btq/WCEBUV1Wg527ZtS3FxcY3n4+yt9HPltOOaa65h69atNdbx6aef4uHhQffu3S963uaa4fDhw+zatYtHH32UYcOGcc0113D06NEaWb788ktOnjzpWnbmpRBg7/9Fzu3uu+9m9erVrvcpT3E6ncyZM4eePXvWen/zTD179qSysrLG/5M0PRXmWQ4fPkxubi7ff/897777LvPmzeOGG24gKCgIy7J45ZVXyMrK4pe//CWbNm1i3759vPHGG/zsZz8jKSnJ9csEVq5cya9//WtWrlzJ7t272bVrF7NmzWLVqlX8/Oc/v6SPISYmhoqKCubPn8+3337L//7v/9Y6GWXy5Ml8/vnn/P73v+fLL79k586dLFy4kIKCAgIDA/nDH/7Agw8+yOLFi/nmm2/YvHkz8+fPZ/HixY2Ws3///vj7+zNlyhS++eYblixZwquvvlpjzLRp03jttdeYNm0aO3bs4KuvvuLpp5+2vY5f/epX+Pr6MnLkSLZt28ZHH33E+PHjuffee12Hhy+1y5mhTZs2hIaG8tJLL7Fnzx7WrFnDxIkTXV//z//8TyzLYsyYMWRlZbFq1apaRzzs/L80tQULFpz3UP2lduq14szbqYJ78MEHSUxM5Gc/+xlvvPEG+/bt4/PPP+eXv/wlO3bs4JVXXnEdbRk6dCgvvvgiGRkZZGdns2rVKqZMmeJ63ZHmQ4V5luTkZNq1a0d0dDT3338/t9xyC0uXLnV9fcCAAWzYsAFPT09uvvlmYmJimDx5MiNHjuTDDz/Ex8cHqN5C9Pf356GHHiIuLo7rr7+ef/7zn/ztb3/j3nvvvaSPITY2lr/85S88/fTT9OrVi3/84x+kpqbWGNOtWzc++OADtm7dSmJiIklJSaxYsYJWraovzX3iiSf485//TGpqKtdccw0/+clPeOedd+jSpUuj5QwJCeH//u//WLVqFb179+a1115j+vTpNcYMHTqUN954g7fffpu4uDh+/OMfX9AF3P7+/rz//vscOXKEfv36cccddzBs2DAWLFjQaI+jOWXw8PDg9ddfJyMjg169evHggw/y7LPPur4eEBDAv//9b7766iv69u3LI488UmsDxM7/S1MrKCg47/vpl9qp14ozb6d+u5Ovry9r1qxhxIgRTJkyhZiYGH7yk5/g6enJhg0bXNdgAgwfPpzFixdz0003cc011zB+/HiGDx/OP//5zyZ6ZFIfy9h9Z/oKM3ToUOLi4i7rr1Wrz/Tp03nrrbd02EuaRHZ2Nl26dGHLli0t5tfdXYjm8lqh/6em16L3MJ9//nkCAgIa7azVC7Vv3z4CAgKYMWNGk6xfROxp6teKm2++udZvFZLLr8XuYe7fv58TJ04A0KlTJ7y9vS97hsrKSrKzswHw8fFp1BNqROzSnsu5NYfXiuaQQVpwYYqIiFyIFn1IVkRExC4VpoiIiA0qTBERERtUmCIiIjaoMBuorKyM6dOnU1ZW1tRRbHO3zMp7ablbXnC/zO6WV+qms2QbqKioCIfDQWFhodv8Git3y6y8l5a75QX3y+xueaVu2sMUERGxQYUpIiJiQ6umDnC5OJ1ODhw4QGBgoOuvBDSGoqKiGv+6A3fLrLyXlrvlBffLfKnyGmMoLi6mffv2eHho/+dSazHvYX7//ff61XMickXKycmhY8eOTR3jitdi9jADAwMB2Ls5mqAA99gS+0W33k0dQUSasUoqWMcq1+ubXFotpjBPHYYNCvAgKNA9CrOV5dXUEUSkOfvh+GBjvs0k9XOP5hAREWliKkwREREbVJgiIiI2qDBFRERsUGGKiIjYoMIUERGxQYUpIiJigwpTRETEBhWmiIiIDSpMERERG1SYIiIiNqgwRUREbFBhioiI2ODWf61k6NChxMXFMXfu3Ead9/lFx5j1/DFy86uI7enNvKfaktjXt86xL/9fIf/3RjHbdpUDEN/Hhycnh9YY79luT533ffrPofzh920anDfH7GEvuynnJAE46E5fHFZInWP3m285yF6OU/2HbINow1X0qjF+tflXnfeNoTfRVvcWl9cdMyuvvifO9txzz/Hss8+Sm5tLbGws8+fPJzExsc6xL7/8Mn//+9/Ztm0bAPHx8cyYMaPG+Pr+QsozzzzDH//4xysyr1v/AekjR47g5eVl62/BFRUV4XA4OLq76zn/vNfSFcWM+u88nn86nP59fZn38jH+tfI4O9Z1Ijys9vbFr3+fy8BEP5ISfPH1sXjmuaO89W4JX63tRId21eNzD1XWuM+7a0oZM/EQu9d3pmvn+v+E1/D2ced9XLkmh+18zjVcRxAh5PA1eXzPAIbjbdUu+W1mIw7CCCYUDzzIZhf5HOB6bsLX8gOgzJyscZ/D5JLFFwzgJ/hbAefNdCXldcfMyttyvicqTQVrWUFhYSFBQUH15l26dCkjRozghRdeoH///sydO5c33niDXbt2ER4eXmv8r371KwYOHMiAAQPw9fXl6aefZvny5Wzfvp0OHTpUPwe5uTXu8+6773LfffexZ88eunbteu4n8Dyaa163LswLYbcwk27JISHOl/kz2gLgdBo6x2cz7jfBPDz+/HuDVVWG0B7f8ten2jLiP+r+Bv7FqIMcL3Hy4RsdzjmXncLcZNIIIoQeVl8AjDGs4x2iiCHa6nHe+xtjWMsKutOX9lbnOsdsNZ9RSQXx1pDzznel5XXHzMp7afM2p8x2C7N///7069ePBQsWAOB0OomKimL8+PFMmjTpvHmrqqpo06YNCxYsYMSIEXWOue222yguLiYtLe28851Pc83r1u9hDh06lJSUlEabr7zckPFlGcMG+bmWeXhYDBvkz/qMk+e452mlJwwVlRDSxrPOr+flV7IqrYTR99T/zW2X0zgp5hghnN7isiyLECI4xmFbc1RRicGJF3Xv6ZaZkxRwkA50aXF53TGz8l7avO6Yuby8nIyMDJKTk13LPDw8SE5OZv369bbmKC0tpaKigpCQug855+Xl8c4773Dfffdd0XndujAbW8GRKqqqIKJtzbKLaOtJ3lmHVesz6ckC2kd4knxG6Z7p7/8sJjDAg9tvad3gvBWUYTB4U/MQkDc+lGOv4PfwFT74EUJEnV8/yF48aUVbzr03fCXmBffLrLyXNi+4X+aCggKqqqqIiKi5roiIiFqHKevz8MMP0759+xoldqbFixcTGBjI7bfffkXndeuTfs6lrKyMsrIy1+dFRUWXfJ1Pzz/K0hXHWfNmB3x9694WWfRaEf95e2C9X7+css1OcskhniF4WnXvER8gm0g61fv1y8nd8oL7ZVbeS8/dMs+cOZPXX3+dtWvX4utb98mP//M//8OvfvWrer9+OV3KvE3/qn2JpKam4nA4XLeoqKjz3icsxBNPT8jLr6qxPC+/iojwc29bzF54lKcXHOW919rTp6dPnWM+2XCCXd9UcN9/NvxwLIAXPlhYtbZqyymrtfV7tr1mF9ns4joGEWgF1znmqMmnlOJGO5TlbnnB/TIr76XNC+6XOSwsDE9PT/Ly8mosz8vLIzIy8pz3nTVrFjNnzuSDDz6gT58+dY755JNP2LVrF7/97W+v+LxXbGFOnjyZwsJC1y0nJ+e89/H2tojv48OadSdcy5xOw5p1pSTF1/+D8OxzR3lyzlFWLWlPQlz94/7ntSLi+/gQe23dhXqhPCwPAgnmCIdcy4wxHOEQwYTWe79ss4tv2UFffkRQPafBQ/VWbiBt6v3BvtLzumNm5b20ed0xs7e3N/Hx8TVObnE6naSlpZGUlFTv/Z555hmeeOIJ3nvvPRISEuod98orrxAfH09sbOwVn/eKLUwfHx+CgoJq3OxI+V0wf/tHEYv/WcSO3eX8/uF8SkoNo+6uvnRl5Pg8pjxV4Br/zIKjTH3mMH/7SzjRUa3IPVRJ7qFKjpc4a8xbVOzkX/8+zm8aae/ylE504wDfccBkU2KK2MlmqqikHdEAbDOb2GO+co3PNjv5hu30JAFfWlNmTlJmTlJpar5HW2kqyON7OvwwT0vN646ZlffS5nXHzBMnTuTll19m8eLF7NixgwceeICSkhJGjx4NwIgRI5g8ebJr/NNPP82f//xn/ud//ofo6Ghyc3PJzc3l+PHjNeYtKirijTfeaLS9y+ae94p9D/Ni3fXzQAoOVzH9mSPk5lcSd60Pq5a0J6Jt9VOVs78CjzM2M15YXEh5OfzHmJpvRk99qA3T/nB6a/P1t4oxBu75RcOvATtTpBVFhSnjW7Io4ySBOOjLj/D54Vqwk5RicfqC3e/5FoOTr9hQY54uXMNVXOv6PJfqPfJIOrXovO6YWXkvbV53zHzXXXeRn5/P1KlTyc3NJS4ujvfee891Ys2+ffvwOOOFbeHChZSXl3PHHXfUmGfatGlMnz7d9fnrr7+OMYZ77rmnReR16+swL+Q3/di9DrM5sXMdpoi0XHavw5TG4dZ7mGvXrm3qCCIi0kK4x66WiIhIE1NhioiI2KDCFBERsUGFKSIiYoMKU0RExAYVpoiIiA0qTBERERtUmCIiIjaoMEVERGxQYYqIiNigwhQREbFBhSkiImKDClNERMQGFaaIiIgNKkwREREbVJgiIiI2qDBFRERsUGGKiIjYoMIUERGxQYUpIiJigwpTRETEBhWmiIiIDSpMERERG1SYIiIiNqgwRUREbFBhioiI2KDCFBERsUGFKSIiYoMKU0RExAYVpoiIiA0qTBERERtUmCIiIja0auoAzdHzi44x6/lj5OZXEdvTm3lPtSWxr2+dY7fvKmPaM0fY/GUZe7+v5C+PhTHh/uAaY2b+9QjLV5Wwc085fr4eJCX4MvPRULrHeDdK3hyzh73sppyTBOCgO31xWCF1jj1uCvmGLIo5yklK6UYsnayra4z5zuwkn/2UUIwHngQTSgy9aW0Ftsi87phZeZX3bM899xzPPvssubm5xMbGMn/+fBITE+scu337dqZOnUpGRgZ79+5lzpw5pKSk1BiTmprKsmXL2LlzJ35+fgwYMICnn36a7t27X7F5tYd5lqUrinloegF/fiiEL96Pok9PH26+5wCHCirrHF96wtC1sxczHgklMtyzzjHp60/ywGgHn73TkfeXtqei0vCTuw9QUupscN5ck8NuvqQrPUkkmUCC2cInlJuTdY6vogp/WhNDb7ypeyPgGPl05Cr6cQPXMQgnTrbwCVWm7ufgSs7rjpmVV3nPtnTpUiZOnMi0adPYvHkzsbGxDB8+nEOHDtU5vrS0lK5duzJz5kwiIyPrHJOens7YsWPZsGEDH374IRUVFdx0002UlJRcsXktY4y5qEfkZoqKinA4HBzd3ZWgwPq3E5JuySEhzpf5M9oC4HQaOsdnM+43wTw8vs0519G1XzYTxgTX2sM8W35BFZG9v+OjZR0YnORX77jh7ePOOQ/AJpNGECH0sPoCYIxhHe8QRQzRVo9z3nedWUUnrq61tXu2clPGx/ybeIbQxmp73kxXUl53zKy8LSdvpalgLSsoLCwkKCio3nH9+/enX79+LFiwAACn00lUVBTjx49n0qRJ58wSHR1NSkpKrT22s+Xn5xMeHk56ejqDBw8+59jzaa55tYd5hvJyQ8aXZQwbdLrEPDwshg3yZ31G3VuPF6OwuAqAkDYNe/qdxkkxxwgh3LXMsixCiOAYhxs095kqqQDAi4YdQna3vOB+mZW3bi01L0B5eTkZGRkkJye7lnl4eJCcnMz69esbPP8phYWFAISE1H1o2q7mnLdZFubQoUMZP348KSkptGnThoiICF5++WVKSkoYPXo0gYGBxMTE8O677zbqeguOVFFVBRFtax5ajWjrSd6hxjk04nQaHpxawMB+vvTq4dOguSoow2BqHebxxodyGqfgjTHsJhMHoQRYjgbN5W55wf0yK29tLTkvQEFBAVVVVURERNRYHhERQW5uboPnh+o9wJSUFAYOHEivXr0aNFdzztssCxNg8eLFhIWFsWnTJsaPH88DDzzAnXfeyYABA9i8eTM33XQT9957L6WlpXXev6ysjKKiohq35mDc5Hy27yxnyQt1H2dvbnayheMU0Zv+TR3FFnfLC+6XWXkvLXfLCzB27Fi2bdvG66+/3tRRbLnYvM22MGNjY3n00Ue5+uqrmTx5Mr6+voSFhTFmzBiuvvpqpk6dyuHDh/nyyy/rvH9qaioOh8N1i4qKOu86w0I88fSEvPyqGsvz8quICG/4CcXjp+TzzupS0t7sQMf2DZ/PCx8srFpbtuWU1XtywYXYabZQwEHiGYKv5d/g+dwtL7hfZuWtqaXnBQgLC8PT05O8vLway/Py8uo9QeZCjBs3jpUrV/LRRx/RsWPHBs/XnPM228Ls06eP62NPT09CQ0Pp3bu3a9mp3fX6zpqaPHkyhYWFrltOTs551+ntbRHfx4c16064ljmdhjXrSkmKv/gfBmMM46fk89a7x1n9Rnu6dPK66LnO5GF5EEgwRzj9HBhjOMIhggm96HmNMew0W8hnP/EMxs9q3Rhx3S4vuF9m5T09h/JW8/b2Jj4+nrS0NNcyp9NJWloaSUlJFz2vMYZx48axfPly1qxZQ5cuXRojbrPO22yvw/TyqlkqlmXVWGZZFlD9RNbFx8cHH58Lf48w5XfBjJ5wiPhYHxLjfJn38jFKSg2j7q6+Hmrk+Dw6RHoy45EwoPpEoazd5dUfVxj251aSua2MgNYWMV2q37AfNzmf15YfZ/midgQGeJD7w/uhjkAP/Pwats3SiW5k8TlBpg0OQtjH11RRSTuiAdhmNuGLHzFW9caG0zgpofrwtBMnZZyg2BzDk1b4WwEA7GILueQQywA88aLsh9PlW+GFp1X3pTNXal53zKy8ynu2iRMnMnLkSBISEkhMTGTu3Lmuc0IARowYQYcOHUhNTQWqT7zJyspyfbx//34yMzMJCAggJiYGqD6suWTJElasWEFgYKDr/UWHw4GfX/1n/7tz3mZbmE3lrp8HUnC4iunPHCE3v5K4a31YtaQ9EW2rn6qc/RV4nNFxB/Iqib/x9N7r7IXHmL3wGEOSfFmzrHp3/4XF1T8sP/7l/hrremVuOKPuqv9UcDsirSgqTBnfkkUZJwnEQV9+hI9VvUd8klIsLNf4Mk6wkdWuz/eym73sJpgwEhgKwPd8C0AG6TXW1ZME2v/wotBS8rpjZuVV3rPddddd5OfnM3XqVHJzc4mLi+O9995zHanbt28fHme8sB04cIC+ffu6Pp81axazZs1iyJAhrF27FoCFCxcC1SdpnmnRokWMGjXqiszbLK/DHDp0KHFxccydO9e1rK5rayzLYvny5dx2223nndPudZjNiZ3rMEWk5bJ7HaY0DvdoDhERkSbWLA/JntqFPlN2dnatZc1w51hERK5Q2sMUERGxQYUpIiJigwpTRETEBhWmiIiIDSpMERERG1SYIiIiNqgwRUREbFBhioiI2KDCFBERsUGFKSIiYoMKU0RExAYVpoiIiA0qTBERERtUmCIiIjaoMEVERGxQYYqIiNigwhQREbFBhSkiImKDClNERMQGFaaIiIgNKkwREREbVJgiIiI2qDBFRERsUGGKiIjYoMIUERGxQYUpIiJigwpTRETEBhWmiIiIDSpMERERG1SYIiIiNqgwRUREbGjV1AGao+cXHWPW88fIza8itqc3855qS2Jf3zrHbt9VxrRnjrD5yzL2fl/JXx4LY8L9wTXGzPzrEZavKmHnnnL8fD1ISvBl5qOhdI/xbpS8OWYPe9lNOScJwEF3+uKwQuoce9wU8g1ZFHOUk5TSjVg6WVfXGPOd2Uk++ymhGA88CSaUGHrT2gpskXndMbPyKu/ZnnvuOZ599llyc3OJjY1l/vz5JCYm1jl2+/btTJ06lYyMDPbu3cucOXNISUmpMSY1NZVly5axc+dO/Pz8GDBgAE8//TTdu3e/YvNe9j3MV199leDg4Mu9WtuWrijmoekF/PmhEL54P4o+PX24+Z4DHCqorHN86QlD185ezHgklMhwzzrHpK8/yQOjHXz2TkfeX9qeikrDT+4+QEmps8F5c00Ou/mSrvQkkWQCCWYLn1BuTtY5vooq/GlNDL3xpu6NgGPk05Gr6McNXMcgnDjZwidUmbqfgys5rztmVl7lPdvSpUuZOHEi06ZNY/PmzcTGxjJ8+HAOHTpU5/jS0lK6du3KzJkziYyMrHNMeno6Y8eOZcOGDXz44YdUVFRw0003UVJScsXmtYwx5qIe0UU6ceIExcXFhIeHX87VUlRUhMPh4OjurgQF1r+dkHRLDglxvsyf0RYAp9PQOT6bcb8J5uHxbc65jq79spkwJrjWHubZ8guqiOz9HR8t68DgJL96xw1vH3fOeQA2mTSCCKGH1RcAYwzreIcoYoi2epzzvuvMKjpxda2t3bOVmzI+5t/EM4Q2VtvzZrqS8rpjZuVtOXkrTQVrWUFhYSFBQUH1juvfvz/9+vVjwYIFADidTqKiohg/fjyTJk06Z5bo6GhSUlJq7bGdLT8/n/DwcNLT0xk8ePA5x55Pc8172fcw/fz8LntZ2lVebsj4soxhg06XmIeHxbBB/qzPqHvr8WIUFlcBENKmYU+/0zgp5hghnH4+LcsihAiOcbhBc5+pkgoAvGjYIWR3ywvul1l569ZS8wKUl5eTkZFBcnKya5mHhwfJycmsX7++wfOfUlhYCEBISN2Hpu1qznkv+BW7uLiYX/3qV7Ru3Zp27doxZ84chg4d6mrzo0ePMmLECNq0aYO/vz8333wzX3/9tev+Zx+SnT59OnFxcfzv//4v0dHROBwO7r77boqLi22vs7EUHKmiqgoi2tY8tBrR1pO8Q41zaMTpNDw4tYCB/Xzp1cOnQXNVUIbB1DrM440P5TROwRtj2E0mDkIJsBwNmsvd8oL7ZVbe2lpyXoCCggKqqqqIiIiosTwiIoLc3NwGzw/Ve4ApKSkMHDiQXr16NWiu5pz3ggtz4sSJfPrpp7z99tt8+OGHfPLJJ2zevNn19VGjRvHFF1/w9ttvs379eowx3HLLLVRUVNQ75zfffMNbb73FypUrWblyJenp6cycOdP2OutSVlZGUVFRjVtzMG5yPtt3lrPkhbqPszc3O9nCcYroTf+mjmKLu+UF98usvJeWu+UFGDt2LNu2beP1119v6ii2XGzeCzpLtri4mMWLF7NkyRKGDRsGwKJFi2jfvj0AX3/9NW+//TaffvopAwYMAOAf//gHUVFRvPXWW9x55511zut0Onn11VcJDKw+I+zee+8lLS2Np5566rzrrE9qaiqPPfbYhTw8wkI88fSEvPyqGsvz8quICG/4CcXjp+TzzupS1i7vQMf2DZ/PCx8srFpbtuWU1XtywYXYabZQwEESGIqv5d/g+dwtL7hfZuWtqaXnBQgLC8PT05O8vLway/Py8uo9QeZCjBs3jpUrV/Lxxx/TsWPHBs/XnPNe0B7mt99+S0VFRY1Tex0Oh+u03B07dtCqVSv69z+9ZRQaGkr37t3ZsWNHvfNGR0e7yhKgXbt2rrOhzrfO+kyePJnCwkLXLScn57yPz9vbIr6PD2vWnXAtczoNa9aVkhR/8T8MxhjGT8nnrXePs/qN9nTp5HXRc53Jw/IgkGCOcPrMMWMMRzhEMKEXPa8xhp1mC/nsJ57B+FmtGyOu2+UF98usvKfnUN5q3t7exMfHk5aW5lrmdDpJS0sjKSnpouc1xjBu3DiWL1/OmjVr6NKlS2PEbdZ5m8V1mF5eNQvEsiyczoZdcuHj44OPz4W/R5jyu2BGTzhEfKwPiXG+zHv5GCWlhlF3Vxf6yPF5dIj0ZMYjYUD1iUJZu8urP64w7M+tJHNbGQGtLWK6VL9hP25yPq8tP87yRe0IDPAg94f3Qx2BHvj5NezEn050I4vPCTJtcBDCPr6mikraEQ3ANrMJX/yIsXoD1SctlFB9eNqJkzJOUGyO4Ukr/K0AAHaxhVxyiGUAnnhR9sPp8q3wwtOq+9KZKzWvO2ZWXuU928SJExk5ciQJCQkkJiYyd+5cSkpKGD16NAAjRoygQ4cOpKamAtUn3mRlZbk+3r9/P5mZmQQEBBATEwNUH9ZcsmQJK1asIDAw0PX+osPhwM+v/rP/3TnvBRVm165d8fLy4vPPP6dTp05A9ZlGu3fvZvDgwVxzzTVUVlayceNG1yHZw4cPs2vXLnr27Hkhq7K9zsZ2188DKThcxfRnjpCbX0nctT6sWtKeiLbVT1XO/go8zui4A3mVxN94eu919sJjzF54jCFJvqxZVr27/8Li6h+WH/9yf411vTI3nFF31X8quB2RVhQVpoxvyaKMkwTioC8/wseq3iM+SSkWlmt8GSfYyGrX53vZzV52E0wYCQwF4Hu+BSCD9Brr6kkC7X94UWgped0xs/Iq79nuuusu8vPzmTp1Krm5ucTFxfHee++5TqzZt28fHme8sB04cIC+ffu6Pp81axazZs1iyJAhrF27FoCFCxcCMHTo0BrrWrRoEaNGjboi817wdZhjxowhLS2NV155hfDwcKZNm8YHH3zAfffdx5w5c7jtttv4+uuvefHFFwkMDGTSpEns2bOHrKwsvLy8ePXVV0lJSeHYsWNA9Vmyb731FpmZma51zJ07l7lz55KdnW1rnXbYvQ6zObFzHaaItFx2r8OUxnHBzfGXv/yFpKQkbr31VpKTkxk4cCDXXHMNvr7VW1eLFi0iPj6eW2+9laSkJIwxrFq1qtZh18Zcp4iIyKXW4N/0U1JSQocOHZg9ezb33XdfY+Vq9HVqD1NErjTaw7y8Lvikny1btrBz504SExMpLCzk8ccfB+DnP/95o4drynWKiIic6aLOkp01axa7du1ynf77ySefEBYW1tjZmnydIiIip1z2X77eVHRIVkSuNDoke3m5R3OIiIg0MRWmiIiIDSpMERERG1SYIiIiNqgwRUREbFBhioiI2KDCFBERsUGFKSIiYoMKU0RExAYVpoiIiA0qTBERERtUmCIiIjaoMEVERGxQYYqIiNigwhQREbFBhSkiImKDClNERMQGFaaIiIgNKkwREREbVJgiIiI2qDBFRERsUGGKiIjYoMIUERGxQYUpIiJigwpTRETEBhWmiIiIDSpMERERG1SYIiIiNqgwRUREbGjV1AGao+cXHWPW88fIza8itqc3855qS2Jf3zrHbt9VxrRnjrD5yzL2fl/JXx4LY8L9wTXGzPzrEZavKmHnnnL8fD1ISvBl5qOhdI/xbpS8OWYPe9lNOScJwEF3+uKwQuoce9wU8g1ZFHOUk5TSjVg6WVfXGPOd2Uk++ymhGA88CSaUGHrT2gpskXndMbPyKu/ZnnvuOZ599llyc3OJjY1l/vz5JCYm1jl2+/btTJ06lYyMDPbu3cucOXNISUmpMSY1NZVly5axc+dO/Pz8GDBgAE8//TTdu3e/YvM2yz3MoUOH1nqwl8vSFcU8NL2APz8UwhfvR9Gnpw8333OAQwWVdY4vPWHo2tmLGY+EEhnuWeeY9PUneWC0g8/e6cj7S9tTUWn4yd0HKCl1NjhvrslhN1/SlZ4kkkwgwWzhE8rNyTrHV1GFP62JoTfe1L0RcIx8OnIV/biB6xiEEydb+IQqU/dzcCXndcfMyqu8Z1u6dCkTJ05k2rRpbN68mdjYWIYPH86hQ4fqHF9aWkrXrl2ZOXMmkZGRdY5JT09n7NixbNiwgQ8//JCKigpuuukmSkpKrti8ljHGXNQjuoSGDh1KXFwcc+fObbQ5i4qKcDgcHN3dlaDA+rcTkm7JISHOl/kz2gLgdBo6x2cz7jfBPDy+zTnX0bVfNhPGBNfawzxbfkEVkb2/46NlHRic5FfvuOHt4845D8Amk0YQIfSw+gJgjGEd7xBFDNFWj3Ped51ZRSeurrW1e7ZyU8bH/Jt4htDGanveTFdSXnfMrLwtJ2+lqWAtKygsLCQoKKjecf3796dfv34sWLAAAKfTSVRUFOPHj2fSpEnnzBIdHU1KSsp5d2Ly8/MJDw8nPT2dwYMHn3Ps+TTXvM1uD3PUqFGkp6czb948LMvCsiyys7NJT08nMTERHx8f2rVrx6RJk6isbJytr1PKyw0ZX5YxbNDpEvPwsBg2yJ/1GXVvPV6MwuIqAELaNOzpdxonxRwjhHDXMsuyCCGCYxxu0NxnqqQCAC8adgjZ3fKC+2VW3rq11LwA5eXlZGRkkJyc7Frm4eFBcnIy69evb/D8pxQWFgIQElL3oWm7mnPeZleY8+bNIykpiTFjxnDw4EEOHjyIl5cXt9xyC/369WPr1q0sXLiQV155hSeffLJR111wpIqqKohoW/PQakRbT/IONU45O52GB6cWMLCfL716+DRorgrKMJhah3m88aGcxil4Ywy7ycRBKAGWo0FzuVtecL/MyltbS84LUFBQQFVVFRERETWWR0REkJub2+D5oXoPMCUlhYEDB9KrV68GzdWc8za7k34cDgfe3t74+/u7jkU/8sgjREVFsWDBAizLokePHhw4cICHH36YqVOn4uFRu/fLysooKytzfV5UVHTZHsO5jJucz/ad5Xy8omNTR7FlJ1s4ThEJDG3qKLa4W15wv8zKe2m5W16AsWPHsm3bNtatW9fUUWy52LzNbg+zLjt27CApKQnLslzLBg4cyPHjx/n+++/rvE9qaioOh8N1i4qKOu96wkI88fSEvPyqGsvz8quICG/4tsX4Kfm8s7qUtDc70LF9w+fzwgcLq9aWbTll9Z5ccCF2mi0UcJB4huBr+Td4PnfLC+6XWXlraul5AcLCwvD09CQvL6/G8ry8vHpPkLkQ48aNY+XKlXz00Ud07NjwHYHmnNctCvNiTJ48mcLCQtctJyfnvPfx9raI7+PDmnUnXMucTsOadaUkxV/8D4MxhvFT8nnr3eOsfqM9XTp5XfRcZ/KwPAgkmCOcPnPMGMMRDhFM6EXPa4xhp9lCPvuJZzB+VuvGiOt2ecH9Mivv6TmUt5q3tzfx8fGkpaW5ljmdTtLS0khKSrroeY0xjBs3juXLl7NmzRq6dOnSGHGbdd5md0gWqp+wqqrTe3nXXHMNb775JsYY117mp59+SmBgYL1bCD4+Pvj4XPh7hCm/C2b0hEPEx/qQGOfLvJePUVJqGHV39fVQI8fn0SHSkxmPhAHVJwpl7S6v/rjCsD+3ksxtZQS0tojpUv2G/bjJ+by2/DjLF7UjMMCD3B/eD3UEeuDn17Btlk50I4vPCTJtcBDCPr6mikraEQ3ANrMJX/yIsXoD1SctlFB9eNqJkzJOUGyO4Ukr/K0AAHaxhVxyiGUAnnhR9sPp8q3wwtOq+9KZKzWvO2ZWXuU928SJExk5ciQJCQkkJiYyd+5cSkpKGD16NAAjRoygQ4cOpKamAtUn3mRlZbk+3r9/P5mZmQQEBBATEwNUH9ZcsmQJK1asIDAw0PX+osPhwM+v/rP/3Tlvs7ys5P777yczM5N//vOfBAQEUFZWRrdu3Rg9ejTjxo1j165d/Pa3v2Xs2LFMnz7d1px2LysBeO5/Tv3igkrirvVh7pNt6X9d9R7mj2//ns5RXiyaV/2GdHZOBVcl7q01x5AkX9Ysqy5zz3Z76lzPK3PDGXVX/aeC27msBE5fRF3GSQJx0J04HFb11u4XZi1+tOZaqx8AJ0wJn/JurTmCCSPBGgrAavOvOtfTkwTaW9G2Ml1Jed0xs/K2jLx2LysBWLBggesXAcTFxfHXv/6V/v37A9WX8kVHR/Pqq68CkJ2dXece2JAhQ1i7di1AjbfIzrRo0SJGjRp1zix2NMe8zbIwd+/ezciRI9m6dSsnTpzgu+++Y+/evfzxj39k69athISEMHLkSJ588klatbK3k3whhdlc2C1MEWmZLqQwpeGa5SHZbt261breJjo6mk2bNjVRIhERaencY1dLRESkiakwRUREbFBhioiI2KDCFBERsUGFKSIiYoMKU0RExAYVpoiIiA0qTBERERtUmCIiIjaoMEVERGxQYYqIiNigwhQREbFBhSkiImKDClNERMQGFaaIiIgNKkwREREbVJgiIiI2qDBFRERsUGGKiIjYoMIUERGxQYUpIiJigwpTRETEBhWmiIiIDSpMERERG1SYIiIiNqgwRUREbFBhioiI2KDCFBERsUGFKSIiYoMKU0RExAYVpoiIiA0qTBERERtaNXWA5uj5RceY9fwxcvOriO3pzbyn2pLY17fOsdt3lTHtmSNs/rKMvd9X8pfHwphwf3CNMTP/eoTlq0rYuaccP18PkhJ8mfloKN1jvBslb47Zw152U85JAnDQnb44rJA6xx43hXxDFsUc5SSldCOWTtbVNcZ8Z3aSz35KKMYDT4IJJYbetLYCW2Red8ysvMp7tueee45nn32W3NxcYmNjmT9/PomJiXWO3b59O1OnTiUjI4O9e/cyZ84cUlJSaoxJTU1l2bJl7Ny5Ez8/PwYMGMDTTz9N9+7dr9i8Tb6HuXbtWizL4tixY00dBYClK4p5aHoBf34ohC/ej6JPTx9uvucAhwoq6xxfesLQtbMXMx4JJTLcs84x6etP8sBoB5+905H3l7anotLwk7sPUFLqbHDeXJPDbr6kKz1JJJlAgtnCJ5Sbk3WOr6IKf1oTQ2+8qXsj4Bj5dOQq+nED1zEIJ0628AlVpu7n4ErO646ZlVd5z7Z06VImTpzItGnT2Lx5M7GxsQwfPpxDhw7VOb60tJSuXbsyc+ZMIiMj6xyTnp7O2LFj2bBhAx9++CEVFRXcdNNNlJSUXLF5LWOMuahHdJGGDh1KXFwcc+fOBaoL84YbbuDo0aMEBwdfsvUWFRXhcDg4ursrQYH1byck3ZJDQpwv82e0BcDpNHSOz2bcb4J5eHybc66ja79sJowJrrWHebb8gioie3/HR8s6MDjJr95xw9vHnXMegE0mjSBC6GH1BcAYwzreIYoYoq0e57zvOrOKTlxda2v3bOWmjI/5N/EMoY3V9ryZrqS87phZeVtO3kpTwVpWUFhYSFBQUL3j+vfvT79+/ViwYAEATqeTqKgoxo8fz6RJk86ZJTo6mpSUlFp7bGfLz88nPDyc9PR0Bg8efM6x59Nc8zb5HmZzUl5uyPiyjGGDTpeYh4fFsEH+rM+oe+vxYhQWVwEQ0qZhT7/TOCnmGCGEu5ZZlkUIERzjcIPmPlMlFQB40bBDyO6WF9wvs/LWraXmBSgvLycjI4Pk5GTXMg8PD5KTk1m/fn2D5z+lsLAQgJCQug9N29Wc817Wwhw1ahTp6enMmzcPy7KwLIvs7GwAMjIySEhIwN/fnwEDBrBr164a912xYgXXXXcdvr6+dO3alccee4zKysY5XHFKwZEqqqogom3NQ6sRbT3JO9Q463I6DQ9OLWBgP1969fBp0FwVlGEwtQ7zeONDOY1T8MYYdpOJg1ACLEeD5nK3vOB+mZW3tpacF6CgoICqqioiIiJqLI+IiCA3N7fB80P1HmBKSgoDBw6kV69eDZqrOee9rIU5b948kpKSGDNmDAcPHuTgwYNERUUB8MgjjzB79my++OILWrVqxW9+8xvX/T755BNGjBjBhAkTyMrK4sUXX+TVV1/lqaeeqnddZWVlFBUV1bg1B+Mm57N9ZzlLXqj7OHtzs5MtHKeI3vRv6ii2uFtecL/MyntpuVtegLFjx7Jt2zZef/31po5iy8XmvayF6XA48Pb2xt/fn8jISCIjI/H0rN6be+qppxgyZAg9e/Zk0qRJfPbZZ5w8Wb3F9thjjzFp0iRGjhxJ165dufHGG3niiSd48cUX611XamoqDofDdTtVzOcSFuKJpyfk5VfVWJ6XX0VEeMNPKB4/JZ93VpeS9mYHOrZv+Hxe+GBh1dqyLaes3pMLLsROs4UCDhLPEHwt/wbP5255wf0yK29NLT0vQFhYGJ6enuTl5dVYnpeXV+8JMhdi3LhxrFy5ko8++oiOHTs2eL7mnLfZvIfZp08f18ft2rUDcJ0RtXXrVh5//HECAgJct1N7qaWlpXXON3nyZAoLC123nJyc82bw9raI7+PDmnUnXMucTsOadaUkxV/8D4MxhvFT8nnr3eOsfqM9XTp5XfRcZ/KwPAgkmCOcPnPMGMMRDhFM6EXPa4xhp9lCPvuJZzB+VuvGiOt2ecH9Mivv6TmUt5q3tzfx8fGkpaW5ljmdTtLS0khKSrroeY0xjBs3juXLl7NmzRq6dOnSGHGbdd5mcx2ml9fpErEsC6h+kgCOHz/OY489xu23317rfr6+dReZj48PPj4X/h5hyu+CGT3hEPGxPiTG+TLv5WOUlBpG3V19PdTI8Xl0iPRkxiNhQPWJQlm7y6s/rjDsz60kc1sZAa0tYrpUv2E/bnI+ry0/zvJF7QgM8CD3h/dDHYEe+Pk1bJulE93I4nOCTBschLCPr6miknZEA7DNbMIXP2Ks3kD1SQslVB+eduKkjBMUm2N40gp/KwCAXWwhlxxiGYAnXpT9cLp8K7zwtOq+dOZKzeuOmZVXec82ceJERo4cSUJCAomJicydO5eSkhJGjx4NwIgRI+jQoQOpqalA9Yk3WVlZro/3799PZmYmAQEBxMTEANWHNZcsWcKKFSsIDAx0vb/ocDjw86v/7H93znvZC9Pb25uqqqrzDzzDddddx65du1wP/FK66+eBFByuYvozR8jNryTuWh9WLWlPRNvqpypnfwUeZ3TcgbxK4m88vfc6e+ExZi88xpAkX9Ysq97df2Fx9Q/Lj3+5v8a6Xpkbzqi76j8V3I5IK4oKU8a3ZFHGSQJx0Jcf4WNVb0icpBQLyzW+jBNsZLXr873sZi+7CSaMBIYC8D3fApBBeo119SSB9j+8KLSUvO6YWXmV92x33XUX+fn5TJ06ldzcXOLi4njvvfdcJ9bs27cPjzNe2A4cOEDfvn1dn8+aNYtZs2YxZMgQ1q5dC8DChQuB6ksFz7Ro0SJGjRp1Rea97Ndh3n///WRmZvLPf/6TgIAAvvzyS4YNG1bjOszMzEz69u3Ld999R3R0NO+//z633norjz76KHfccQceHh5s3bqVbdu28eSTT9par93rMJsTO9dhikjLZfc6TGkcl705/vCHP+Dp6UnPnj1p27Yt+/btO+99hg8fzsqVK/nggw/o168f119/PXPmzKFz586XIbGIiEgT7GE2Fe1hisiVRnuYl5d7NIeIiEgTU2GKiIjYoMIUERGxQYUpIiJigwpTRETEBhWmiIiIDSpMERERG1SYIiIiNqgwRUREbFBhioiI2KDCFBERsUGFKSIiYoMKU0RExAYVpoiIiA0qTBERERtUmCIiIjaoMEVERGxQYYqIiNigwhQREbFBhSkiImKDClNERMQGFaaIiIgNKkwREREbVJgiIiI2qDBFRERsUGGKiIjYoMIUERGxQYUpIiJigwpTRETEBhWmiIiIDSpMERERG1o1dYDm6PlFx5j1/DFy86uI7enNvKfaktjXt86x23eVMe2ZI2z+soy931fyl8fCmHB/cI0xM/96hOWrSti5pxw/Xw+SEnyZ+Wgo3WO8GyVvjtnDXnZTzkkCcNCdvjiskDrHHjeFfEMWxRzlJKV0I5ZO1tU1xnxndpLPfkooxgNPggklht60tgJbZF53zKy8ynu25557jmeffZbc3FxiY2OZP38+iYmJdY7dvn07U6dOJSMjg7179zJnzhxSUlJqjElNTWXZsmXs3LkTPz8/BgwYwNNPP0337t2v2LzawzzL0hXFPDS9gD8/FMIX70fRp6cPN99zgEMFlXWOLz1h6NrZixmPhBIZ7lnnmPT1J3lgtIPP3unI+0vbU1Fp+MndBygpdTY4b67JYTdf0pWeJJJMIMFs4RPKzck6x1dRhT+tiaE33tS9EXCMfDpyFf24gesYhBMnW/iEKlP3c3Al53XHzMqrvGdbunQpEydOZNq0aWzevJnY2FiGDx/OoUOH6hxfWlpK165dmTlzJpGRkXWOSU9PZ+zYsWzYsIEPP/yQiooKbrrpJkpKSq7YvJYxxlzUI3IzRUVFOBwOju7uSlBg/dsJSbfkkBDny/wZbQFwOg2d47MZ95tgHh7f5pzr6NovmwljgmvtYZ4tv6CKyN7f8dGyDgxO8qt33PD2ceecB2CTSSOIEHpYfQEwxrCOd4gihmirxznvu86sohNX19raPVu5KeNj/k08Q2hjtT1vpisprztmVt6Wk7fSVLCWFRQWFhIUFFTvuP79+9OvXz8WLFgAgNPpJCoqivHjxzNp0qRzZomOjiYlJaXWHtvZ8vPzCQ8PJz09ncGDB59z7Pk017zawzxDebkh48syhg06XWIeHhbDBvmzPqPurceLUVhcBUBIm4Y9/U7jpJhjhBDuWmZZFiFEcIzDDZr7TJVUAOBFww4hu1tecL/Mylu3lpoXoLy8nIyMDJKTk13LPDw8SE5OZv369Q2e/5TCwkIAQkLqPjRtV3POe1Gv2P/617/o3bs3fn5+hIaGkpycTElJCZ9//jk33ngjYWFhOBwOhgwZwubNm2vc17IsXnzxRW699Vb8/f255pprWL9+PXv27GHo0KG0bt2aAQMG8M0339S434oVK7juuuvw9fWla9euPPbYY1RWNs7hilMKjlRRVQURbWseWo1o60neocZZl9NpeHBqAQP7+dKrh0+D5qqgDIOpdZjHGx/KaZyCN8awm0wchBJgORo0l7vlBffLrLy1teS8AAUFBVRVVREREVFjeUREBLm5uQ2eH6r3AFNSUhg4cCC9evVq0FzNOe8FF+bBgwe55557+M1vfsOOHTtYu3Ytt99+O8YYiouLGTlyJOvWrWPDhg1cffXV3HLLLRQXF9eY44knnmDEiBFkZmbSo0cP/vM//5Pf/e53TJ48mS+++AJjDOPGjXON/+STTxgxYgQTJkwgKyuLF198kVdffZWnnnqq3pxlZWUUFRXVuDUH4ybns31nOUteqPs4e3Ozky0cp4je9G/qKLa4W15wv8zKe2m5W16AsWPHsm3bNl5//fWmjmLLxea94LNkDx48SGVlJbfffjudO3cGoHfv3gD8+Mc/rjH2pZdeIjg4mPT0dG699VbX8tGjR/Mf//EfADz88MMkJSXx5z//meHDhwMwYcIERo8e7Rr/2GOPMWnSJEaOHAlA165deeKJJ/jTn/7EtGnT6syZmprKY489dkGPLSzEE09PyMuvqrE8L7+KiPCGn1A8fko+76wuZe3yDnRs3/D5vPDBwqq1ZVtOWb0nF1yInWYLBRwkgaH4Wv4Nns/d8oL7ZVbemlp6XoCwsDA8PT3Jy8ursTwvL6/eE2QuxLhx41i5ciUff/wxHTt2bPB8zTnvBe9hxsbGMmzYMHr37s2dd97Jyy+/zNGjR4HqBzRmzBiuvvpqHA4HQUFBHD9+nH379tWYo0+fPq6PT+12nyrdU8tOnjzp2ivcunUrjz/+OAEBAa7bmDFjOHjwIKWlpXXmnDx5MoWFha5bTk7OeR+bt7dFfB8f1qw74VrmdBrWrCslKf7ifxiMMYyfks9b7x5n9Rvt6dLJ66LnOpOH5UEgwRzh9JljxhiOcIhgQi96XmMMO80W8tlPPIPxs1o3Rly3ywvul1l5T8+hvNW8vb2Jj48nLS3NtczpdJKWlkZSUtJFz3vqSODy5ctZs2YNXbp0aYy4zTrvBe/meHp68uGHH/LZZ5/xwQcfMH/+fB555BE2btzIAw88wOHDh5k3bx6dO3fGx8eHpKQkysvLa8zh5XW6MCzLqneZ01l92cXx48d57LHHuP3222vl8fWtu8h8fHzw8bnw9whTfhfM6AmHiI/1ITHOl3kvH6Ok1DDq7urroUaOz6NDpCczHgkDqk8Uytpd/fjKKwz7cyvJ3FZGQGuLmC7Vb9iPm5zPa8uPs3xROwIDPMj94f1QR6AHfn4NO/GnE93I4nOCTBschLCPr6miknZEA7DNbMIXP2Ks6g0Sp3FSQvWGiBMnZZyg2BzDk1b4WwEA7GILueQQywA88aLsh9PlW+GFp1X3pTNXal53zKy8ynu2iRMnMnLkSBISEkhMTGTu3LmUlJS4juSNGDGCDh06kJqaClSfeJOVleX6eP/+/WRmZhIQEEBMTAxQfVhzyZIlrFixgsDAQNf7iw6HAz+/+s/+d+e8F3Vc0LIsBg4cyMCBA5k6dSqdO3dm+fLlfPrppzz//PPccsstAOTk5FBQUHAxq6jhuuuuY9euXa4Hfind9fNACg5XMf2ZI+TmVxJ3rQ+rlrQnom31U5WzvwKPMzruQF4l8Tee3nudvfAYsxceY0iSL2uWVe/uv7C4+oflx7/cX2Ndr8wNZ9Rd9Z8KbkekFUWFKeNbsijjJIE46MuP8LGqNyROUoqF5Rpfxgk2str1+V52s5fdBBNGAkMB+J5vAcggvca6epJA+x9eFFpKXnfMrLzKe7a77rqL/Px8pk6dSm5uLnFxcbz33nuuI3z79u3D44wXtgMHDtC3b1/X57NmzWLWrFkMGTKEtWvXArBw4UIAhg4dWmNdixYtYtSoUVdk3gu+DnPjxo2kpaVx0003ER4ezsaNG/n1r3/NW2+9xSOPPEJYWBjz5s2jqKiIP/7xj3zxxRfMmDHDdU2MZVksX76c2267DYDs7Gy6dOnCli1biIuLA2Dt2rXccMMNHD16lODgYN5//31uvfVWHn30Ue644w48PDzYunUr27Zt48knn7SV2+51mM2JneswRaTlsnsdpjSOC26OoKAgPv74Y2655Ra6devGo48+yuzZs7n55pt55ZVXOHr0KNdddx333nsv//3f/014ePj5Jz2P4cOHs3LlSj744AP69evH9ddfz5w5c1wnHYmIiFxq+k0/zZj2MEXkXLSHeXm5R3OIiIg0MRWmiIiIDSpMERERG1SYIiIiNqgwRUREbFBhioiI2KDCFBERsUGFKSIiYoMKU0RExAYVpoiIiA0qTBERERtUmCIiIjaoMEVERGxQYYqIiNigwhQREbFBhSkiImKDClNERMQGFaaIiIgNKkwREREbVJgiIiI2qDBFRERsUGGKiIjYoMIUERGxQYUpIiJigwpTRETEBhWmiIiIDSpMERERG1SYIiIiNqgwRUREbFBhioiI2KDCFBERsUGFKSIiYkOjFubatWuxLItjx47VO2b69OnExcU15mob3fOLjtG1Xzb+0d+QdEsOm7acrHfs9l1l3HHfQbr2y8az3R7mvXSs1piZfz1C/5/k4Ij5hshe3/GLUQfZtae80fLmmD2sM6tYY5axyaRRaI7UO/a4KWSrWc86s4rV5l/sM1/XGvOd2ckmk8ZH5i3Szb/Zaj6jxBS32LzumFl5lfdszz33HNHR0fj6+tK/f382bdpU79jt27fzy1/+kujoaCzLYu7cubXGpKam0q9fPwIDAwkPD+e2225j165dV3TeBhXm0KFDSUlJuaD7/OEPfyAtLa0hq72klq4o5qHpBfz5oRC+eD+KPj19uPmeAxwqqKxzfOkJQ9fOXsx4JJTIcM86x6SvP8kDox189k5H3l/anopKw0/uPkBJqbPBeXNNDrv5kq70JJFkAglmC59Qbuou+Sqq8Kc1MfTGG986xxwjn45cRT9u4DoG4cTJFj6hytT9HFzJed0xs/Iq79mWLl3KxIkTmTZtGps3byY2Npbhw4dz6NChOseXlpbStWtXZs6cSWRkZJ1j0tPTGTt2LBs2bODDDz+koqKCm266iZKSkis2r2WMMRf1iKguzLi4OFebr127lhtuuIGjR48SHBx8sdNeEkVFRTgcDo7u7kpQYP3bCUm35JAQ58v8GW0BcDoNneOzGfebYB4e3+ac6+jaL5sJY4KZcH/wOcflF1QR2fs7PlrWgcFJfvWOG94+7pzzAGwyaQQRQg+rLwDGGNbxDlHEEG31OOd915lVdOJqOllXn3NcuSnjY/5NPENoY7U9b6YrKa87ZlbelpO30lSwlhUUFhYSFBRU77j+/fvTr18/FixYAIDT6SQqKorx48czadKkc2aJjo4mJSXlvDtH+fn5hIeHk56ezuDBg8859nyaa96L3sMcNWoU6enpzJs3D8uysCyL7OxsADIyMkhISMDf358BAwbU2O09+5DsqFGjuO2225g1axbt2rUjNDSUsWPHUlFR4Rpz8OBBfvrTn+Ln50eXLl1YsmQJ0dHRde52N0R5uSHjyzKGDTpdYh4eFsMG+bM+o/7DsheqsLgKgJA2DTsi7jROijlGCOGuZZZlEUIExzjcoLnPVEn1/4UX3g2ax93ygvtlVt66tdS8AOXl5WRkZJCcnOxa5uHhQXJyMuvXr2/w/KcUFhYCEBIS0qB5mnPei37FnjdvHklJSYwZM4aDBw9y8OBBoqKiAHjkkUeYPXs2X3zxBa1ateI3v/nNOef66KOP+Oabb/joo49YvHgxr776Kq+++qrr6yNGjODAgQOsXbuWN998k5deeqneXfNTysrKKCoqqnE7n4IjVVRVQUTbmodWI9p6kneocQ6NOJ2GB6cWMLCfL716+DRorgrKMJhah3m88aGcxil4Ywy7ycRBKAGWo0FzuVtecL/MyltbS84LUFBQQFVVFRERETWWR0REkJub2+D5oXoPMCUlhYEDB9KrV68GzdWc87a62BU6HA68vb3x9/d3HTPeuXMnAE899RRDhgwBYNKkSfz0pz/l5MmT+PrWffy+TZs2LFiwAE9PT3r06MFPf/pT0tLSGDNmDDt37mT16tV8/vnnJCQkAPC3v/2Nq68+9yGN1NRUHnvssYt9eJfMuMn5bN9ZzscrOjZ1FFt2soXjFJHA0KaOYou75QX3y6y8l5a75QUYO3Ys27ZtY926dU0dxZaLzXtJLivp06eP6+N27doBnHOP8Nprr8XT8/ReXbt27Vzjd+3aRatWrbjuuutcX4+JiaFNm3O/nzh58mQKCwtdt5ycnPPmDgvxxNMT8vKraizPy68iIvyity1cxk/J553VpaS92YGO7Rs+nxc+WFi1tmzLKav35IILsdNsoYCDxDMEX8u/wfO5W15wv8zKW1NLzwsQFhaGp6cneXl5NZbn5eXVe4LMhRg3bhwrV67ko48+omPHhu8INOe8l6Qwvby8XB9blgVU7wLbGX/qPucab4ePjw9BQUE1bufj7W0R38eHNetOuJY5nYY160pJir/4HwZjDOOn5PPWu8dZ/UZ7unTyOv+dbPCwPAgkmCOc3hgxxnCEQwQTetHzGmPYabaQz37iGYyf1box4rpdXnC/zMp7eg7lrebt7U18fHyNqxOcTidpaWkkJSVd9LzGGMaNG8fy5ctZs2YNXbp0aYy4zTpvg3ZzvL29qaqqOv/ABujevTuVlZVs2bKF+Ph4APbs2cPRo0cvyfpSfhfM6AmHiI/1ITHOl3kvH6Ok1DDq7kAARo7Po0OkJzMeCQOqTxTK2l19TWV5hWF/biWZ28oIaG0R06X6Dftxk/N5bflxli9qR2CAB7k/vB/qCPTAz69h2yyd6EYWnxNk2uAghH18TRWVtCMagG1mE774EWP1BqpPWiih+v1cJ07KOEGxOYYnrfC3AgDYxRZyySGWAXjiRdkPp8u3wgtPq+5LZ67UvO6YWXmV92wTJ05k5MiRJCQkkJiYyNy5cykpKWH06NFA9XkiHTp0IDU1Fag+8SYrK8v18f79+8nMzCQgIICYmBig+rDmkiVLWLFiBYGBga73Fx0OB35+9Z/97855G1SY0dHRbNy4kezsbAICAhq8V1iXHj16kJyczP3338/ChQvx8vLioYcews/Pz7X32pju+nkgBYermP7MEXLzK4m71odVS9oT0bb6qcrZX4HHGR13IK+S+BtPH+6dvfAYsxceY0iSL2uWVe/uv7C4+oflx7/cX2Ndr8wNZ9Rd59/zPZdIK4oKU8a3ZFHGSQJx0Jcf4WNV7xGfpBSL089TGSfYyGrX53vZzV52E0yY6z2T7/kWgAzSa6yrJwm0/+FFoaXkdcfMyqu8Z7vrrrvIz89n6tSp5ObmEhcXx3vvvec6sWbfvn14nPHCduDAAfr27ev6fNasWcyaNYshQ4awdu1aABYuXAhUX154pkWLFjFq1KgrMm+DrsPcvXs3I0eOZOvWrZw4cYJFixYxevToGtdhZmZm0rdvX7777juio6OZPn06b731FpmZmUD1ZSXHjh3jrbfecs2bkpJCZmam64EePHiQ++67jzVr1hAZGUlqaiopKSk8/vjj/O53v7OV1e51mM2JneswRaTlsnsdpjSOBhVmU/n++++Jiopi9erVDBs2zNZ9VJgicqVRYV5eDT9V8zJYs2YNx48fp3fv3hw8eJA//elPREdHN/i3SYiIiNjlFoVZUVHBlClT+PbbbwkMDGTAgAH84x//qHV2rYiIyKXiFoU5fPhwhg8f3tQxRESkBXOPN/NERESamApTRETEBhWmiIiIDSpMERERG1SYIiIiNqgwRUREbFBhioiI2KDCFBERsUGFKSIiYoMKU0RExAYVpoiIiA0qTBERERtUmCIiIjaoMEVERGxQYYqIiNigwhQREbFBhSkiImKDClNERMQGFaaIiIgNKkwREREbVJgiIiI2qDBFRERsUGGKiIjYoMIUERGxQYUpIiJigwpTRETEBhWmiIiIDSpMERERG1SYIiIiNqgwRUREbFBhioiI2KDCFBERsUGFKSIiYoMKU0RExIZWTR3gUikrK6OsrMz1eVFRUROmERERd3fF7mGmpqbicDhct6ioqKaOJCIibuyKLczJkydTWFjouuXk5DR1JBERcWNX7CFZHx8ffHx8mjqGiIhcIdx2D3PBggUMGzasqWOIiEgL4baFWVBQwDfffNPUMUREpIVw28KcPn062dnZTR1DRERaCLctTBERkctJhSkiImKDClNERMQGFaaIiIgNKkwREREbVJgiIiI2qDBFRERsUGGKiIjYoMIUERGxQYUpIiJigwpTRETEBhWmiIiIDSpMERERG1SYIiIiNqgwRUREbFBhioiI2KDCFBERsUGFKSIiYoMKU0RExAYVpoiIiA0qTBERERtUmCIiIjaoMEVERGxQYYqIiNigwhQREbFBhSkiImKDClNERMQGFaaIiIgNKkwREREbVJgiIiI2qDBFRERsUGGKiIjYoMIUERGx4YIKc+jQoViWhWVZZGZmXqJIzT+DiIi0PBe8hzlmzBgOHjxIr169yM7OdpXX2bcNGza47nPixAmmTZtGt27d8PHxISwsjDvvvJPt27fXmLu0tJTJkydz1VVX4evrS9u2bRkyZAgrVqxwjVm2bBmbNm1qwEMWERG5cK0u9A7+/v5ERkbWWLZ69WquvfbaGstCQ0MBKCsrIzk5mX379jF79mz69+9PXl4eqamp9O/fn9WrV3P99dcD8F//9V9s3LiR+fPn07NnTw4fPsxnn33G4cOHXfOGhIRQVFR0wQ9URESkIS64MOsSGhpaq0RPmTt3LuvXr2fLli3ExsYC0LlzZ95880369+/Pfffdx7Zt27Asi7fffpt58+Zxyy23ABAdHU18fHxjRBQREWmQS37Sz5IlS7jxxhtdZelasYcHDz74IFlZWWzduhWAyMhIVq1aRXFxcYPXW1ZWRlFRUY2biIjIxWqUwhwwYAABAQE1bqfs3r2ba665ps77nVq+e/duAF566SU+++wzQkND6devHw8++CCffvrpRWVKTU3F4XC4blFRURc1j4iICDRSYS5dupTMzMwatzMZY2zNM3jwYL799lvS0tK444472L59O4MGDeKJJ5644EyTJ0+msLDQdcvJybngOURERE5plPcwo6KiiImJqfNr3bp1Y8eOHXV+7dTybt26uZZ5eXkxaNAgBg0axMMPP8yTTz7J448/zsMPP4y3t7ftTD4+Pvj4+FzAoxAREanfJX8P8+6772b16tWu9ylPcTqdzJkzh549e9Z6f/NMPXv2pLKykpMnT17qqCIiIvVqlD3Mw4cPk5ubW2NZcHAwvr6+PPjgg6xYsYKf/exnNS4rmTFjBjt27GD16tVYlgVU/1KCe+65h4SEBEJDQ8nKymLKlCnccMMNBAUFNUZUERGRi9IohZmcnFxr2Wuvvcbdd9+Nr68va9asYcaMGUyZMoW9e/cSGBjIDTfcwIYNG+jVq5frPsOHD2fx4sVMmTKF0tJS2rdvz6233srUqVMbI6aIiMhFa1BhRkdH2zqhx9/fnyeffJInn3zynOMmT57M5MmTGxJJRETkkrjg9zCff/55AgIC+Oqrry5FnvO6+eaba/1WIRERkUvtgvYw//GPf3DixAkAOnXqdEkCnc/f/va3Js8gIiItzwUVZocOHS5VDrfKICIiLY/+HqaIiIgNKkwREREbVJgiIiI2qDBFRERsUGGKiIjYoMIUERGxQYUpIiJigwpTRETEBhWmiIiIDSpMERERG1SYIiIiNjTK38N0B6f+DFnRcWcTJ7Gv0lQ0dQQRacYqqX6NsPNnFqXhWkxhFhcXA9D5uuymDXJBvm3qACLiBoqLi3E4HE0d44pnmRayaeJ0Ojlw4ACBgYFYltVo8xYVFREVFUVOTg5BQUGNNu+l5G6ZlffScre84H6ZL1VeYwzFxcW0b98eDw+9w3aptZg9TA8PDzp27HjJ5g8KCnKLH9wzuVtm5b203C0vuF/mS5FXe5aXjzZJREREbFBhioiI2KDCbCAfHx+mTZuGj49PU0exzd0yK++l5W55wf0yu1teqVuLOelHRESkIbSHKSIiYoMKU0RExAYVpoiIiA0qTBERERtUmCIiIjaoMEVERGxQYYqIiNigwhQREbHh/wNNa32UYV9VmAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 400x514.286 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "i m going to the same thing .\n"
     ]
    }
   ],
   "source": [
    "translator = Translator(model.cpu(), src_tokenizer, trg_tokenizer)\n",
    "result = translator(u'hace mucho frio aqui .')\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "8570b29725840556",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:01:10.140347Z",
     "iopub.status.busy": "2025-01-27T15:01:10.140078Z",
     "iopub.status.idle": "2025-01-27T15:01:10.849895Z",
     "shell.execute_reply": "2025-01-27T15:01:10.849287Z",
     "shell.execute_reply.started": "2025-01-27T15:01:10.140327Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "model = Sequence2Sequence(src_vocab_size=len(src_word2idx), trg_vocab_size=len(trg_word2idx),encoder_num_layers=4,decoder_num_layers=4)\n",
    "model.load_state_dict(torch.load(\"checkpoints/seq2seq_layer_4.ckpt\",weights_only=True, map_location=device))\n",
    "\n",
    "\n",
    "class Translator:\n",
    "    def __init__(self, model, src_tokenizer, trg_tokenizer):\n",
    "        self.model = model\n",
    "        self.model.eval()  # 切换到验证模式\n",
    "        self.src_tokenizer = src_tokenizer\n",
    "        self.trg_tokenizer = trg_tokenizer\n",
    "\n",
    "    def __call__(self, sentence):\n",
    "        # 预处理句子，标点符号处理等\n",
    "        sentence = preprocess_sentence(sentence)\n",
    "        encoder_inputs, attn_mask = self.src_tokenizer.encode(\n",
    "            [sentence.split()],\n",
    "            padding_first=True,\n",
    "            add_bos=True,\n",
    "            add_eos=True,\n",
    "            return_mask=True,\n",
    "        )  # 对输入进行编码，并返回encoder_inputs和encoder_padding_mask\n",
    "        # 转换成tensor\n",
    "        encoder_inputs = torch.Tensor(encoder_inputs).to(dtype=torch.int64)\n",
    "        # 由输入得到预测结果，\n",
    "        # 注意，这里输入的样本只有一个，符合Seq2Seq模型的生成方法要求batch_size=1\n",
    "        preds, scores = self.model.infer(encoder_inputs=encoder_inputs, attn_mask=attn_mask)\n",
    "\n",
    "        # 通过tokenizer转换成文字\n",
    "        # trg_tokenizer.decode方法要求输入的是字符串列表，所以需要[preds]\n",
    "        # 方法返回字符串列表，所以需要[0]\n",
    "        trg_sentence = self.trg_tokenizer.decode([preds], split=True, remove_eos=True)[0]\n",
    "        \n",
    "        # 为了移除句子末尾的特殊标记\n",
    "        return \" \".join(trg_sentence[:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "18b2c2345791f9d5",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:01:10.850887Z",
     "iopub.status.busy": "2025-01-27T15:01:10.850567Z",
     "iopub.status.idle": "2025-01-27T15:10:58.059333Z",
     "shell.execute_reply": "2025-01-27T15:10:58.058762Z",
     "shell.execute_reply.started": "2025-01-27T15:01:10.850864Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/site-packages/nltk/translate/bleu_score.py:577: UserWarning: \n",
      "The hypothesis contains 0 counts of 2-gram overlaps.\n",
      "Therefore the BLEU score evaluates to 0, independently of\n",
      "how many N-gram overlaps of lower order it contains.\n",
      "Consider using lower n-gram order or use SmoothingFunction()\n",
      "  warnings.warn(_msg)\n",
      "/usr/local/lib/python3.10/site-packages/nltk/translate/bleu_score.py:577: UserWarning: \n",
      "The hypothesis contains 0 counts of 3-gram overlaps.\n",
      "Therefore the BLEU score evaluates to 0, independently of\n",
      "how many N-gram overlaps of lower order it contains.\n",
      "Consider using lower n-gram order or use SmoothingFunction()\n",
      "  warnings.warn(_msg)\n",
      "/usr/local/lib/python3.10/site-packages/nltk/translate/bleu_score.py:577: UserWarning: \n",
      "The hypothesis contains 0 counts of 4-gram overlaps.\n",
      "Therefore the BLEU score evaluates to 0, independently of\n",
      "how many N-gram overlaps of lower order it contains.\n",
      "Consider using lower n-gram order or use SmoothingFunction()\n",
      "  warnings.warn(_msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time used: 587.054012298584\n"
     ]
    }
   ],
   "source": [
    "from nltk.translate.bleu_score import sentence_bleu\n",
    "# from torchmetrics.text.bleu import BLEUScore\n",
    "\n",
    "# BLEU 分数的计算原理\n",
    "# n-gram 精度：\n",
    "# BLEU 基于 n-gram（连续的 n 个单词）的匹配程度。\n",
    "# 例如，1-gram 是单个单词，2-gram 是两个连续的单词，依此类推。\n",
    "# 计算候选翻译中每个 n-gram 在参考翻译中出现的次数，并除以候选翻译中该 n-gram 的总数。\n",
    "\n",
    "def evaluate_bleu_on_test_set(test_data, translator):\n",
    "    # 在测试集上计算平均 BLEU 分数。\n",
    "    total_bleu = 0\n",
    "    num_samples = len(test_data)\n",
    "    for src_sentence, ref_translation in test_data:\n",
    "        # 使用翻译器生成翻译结果\n",
    "        candidate_translation = translator(src_sentence)\n",
    "\n",
    "        # 计算 BLEU 分数\n",
    "        bleu_score = sentence_bleu(\n",
    "            [ref_translation.split()], candidate_translation.split(),\n",
    "            weights=(1, 0, 0, 0)\n",
    "        )\n",
    "        total_bleu += bleu_score\n",
    "    avg_bleu = total_bleu / num_samples\n",
    "    return avg_bleu\n",
    "\n",
    "translator=Translator(model, src_tokenizer, trg_tokenizer)\n",
    "\n",
    "start_time = time.time()\n",
    "avg_bleu=evaluate_bleu_on_test_set(test_ds, translator)\n",
    "end_time = time.time()\n",
    "print(\"Time used:\", end_time - start_time)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "460dd740-6aec-4398-a6c6-71ca45ac206a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-27T15:10:58.060451Z",
     "iopub.status.busy": "2025-01-27T15:10:58.060063Z",
     "iopub.status.idle": "2025-01-27T15:10:58.063998Z",
     "shell.execute_reply": "2025-01-27T15:10:58.063155Z",
     "shell.execute_reply.started": "2025-01-27T15:10:58.060425Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "avg_bleu:0.0804895514909054\n"
     ]
    }
   ],
   "source": [
    "print(f\"avg_bleu:{avg_bleu}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
